Реализовано кеширование проверенных плейлистов, при включенном кеше -r теперь не учитывает только некешированные из ini-файла
This commit is contained in:
12
.env.example
12
.env.example
@@ -3,8 +3,10 @@ APP_DEBUG=false
|
|||||||
HTTP_HOST=0.0.0.0
|
HTTP_HOST=0.0.0.0
|
||||||
HTTP_PORT=8031
|
HTTP_PORT=8031
|
||||||
|
|
||||||
REDIS_HOST=
|
CACHE_ENABLED=false
|
||||||
REDIS_PORT=
|
CACHE_HOST=localhost
|
||||||
REDIS_USERNAME=
|
CACHE_PORT=6379
|
||||||
REDIS_PASSWORD=
|
CACHE_USERNAME=
|
||||||
REDIS_DB=
|
CACHE_PASSWORD=
|
||||||
|
CACHE_DB=1
|
||||||
|
CACHE_TTL=1800
|
||||||
|
|||||||
13
app/app.go
13
app/app.go
@@ -7,7 +7,9 @@
|
|||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"axenov/iptv-checker/app/cache"
|
||||||
"axenov/iptv-checker/app/config"
|
"axenov/iptv-checker/app/config"
|
||||||
|
"axenov/iptv-checker/app/logger"
|
||||||
"github.com/redis/go-redis/v9"
|
"github.com/redis/go-redis/v9"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -25,14 +27,15 @@ type Arguments struct {
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
Args Arguments
|
Args Arguments
|
||||||
Redis *redis.Client
|
Cache *redis.Client
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
//TagBlocks []tagfile.TagBlock
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Init инициализирует глобальные переменные
|
// Init инициализирует конфигурацию и подключение к keydb
|
||||||
func Init() {
|
func Init() {
|
||||||
Config = config.Init()
|
Config = config.Init()
|
||||||
//logger.Init(Args.NeedQuiet)
|
logger.Init(Args.NeedQuiet)
|
||||||
//Redis = cache.Init(Config.Redis)
|
if Config.Cache.IsEnabled {
|
||||||
|
Cache = cache.Init(&Config.Cache)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
19
app/cache/cache.go
vendored
19
app/cache/cache.go
vendored
@@ -15,22 +15,23 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
)
|
)
|
||||||
|
|
||||||
func Init(cfg config.RedisConfig) *redis.Client {
|
func Init(cfg *config.CacheConfig) *redis.Client {
|
||||||
rdb := redis.NewClient(&redis.Options{
|
redis := redis.NewClient(&redis.Options{
|
||||||
Addr: fmt.Sprintf("%s:%s", cfg.Host, strconv.Itoa(int(cfg.Port))),
|
Addr: fmt.Sprintf("%s:%s", cfg.Host, strconv.Itoa(int(cfg.Port))),
|
||||||
DB: int(cfg.Db),
|
DB: int(cfg.Db),
|
||||||
PoolSize: 1000,
|
PoolSize: 1000,
|
||||||
ReadTimeout: -1,
|
ReadTimeout: -1,
|
||||||
WriteTimeout: -1,
|
WriteTimeout: -1,
|
||||||
})
|
})
|
||||||
client := rdb.Conn()
|
client := redis.Conn()
|
||||||
|
ctx := context.Background()
|
||||||
var ctx context.Context
|
err := client.Ping(ctx).Err()
|
||||||
if client.Ping(ctx).Err() != nil {
|
if err == nil {
|
||||||
log.Println("Error while connecting to Redis", cfg.Host, cfg.Port, cfg.Db)
|
log.Println("Connected to cache DB")
|
||||||
|
cfg.IsActive = true
|
||||||
} else {
|
} else {
|
||||||
log.Println("Connected to Redis", cfg.Host, cfg.Port, cfg.Db)
|
log.Println("Error while connecting to cache DB, program may work not as expected:", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return rdb
|
return redis
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,9 @@ import (
|
|||||||
"axenov/iptv-checker/app/playlist"
|
"axenov/iptv-checker/app/playlist"
|
||||||
"axenov/iptv-checker/app/tagfile"
|
"axenov/iptv-checker/app/tagfile"
|
||||||
"axenov/iptv-checker/app/utils"
|
"axenov/iptv-checker/app/utils"
|
||||||
|
"context"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"maps"
|
"maps"
|
||||||
@@ -26,7 +28,10 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
var tagBlocks []tagfile.TagBlock
|
var (
|
||||||
|
tagBlocks []tagfile.TagBlock
|
||||||
|
ctx = context.Background()
|
||||||
|
)
|
||||||
|
|
||||||
// PrepareListsToCheck готовит список плейлистов для проверки
|
// PrepareListsToCheck готовит список плейлистов для проверки
|
||||||
func PrepareListsToCheck(files []string, urls []string, codes []string) []playlist.Playlist {
|
func PrepareListsToCheck(files []string, urls []string, codes []string) []playlist.Playlist {
|
||||||
@@ -68,7 +73,19 @@ func PrepareListsToCheck(files []string, urls []string, codes []string) []playli
|
|||||||
lists = append(lists, list)
|
lists = append(lists, list)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
lists = slices.Collect(maps.Values(ini.Lists))
|
if app.Config.Cache.IsActive {
|
||||||
|
cachedLists := getCachedPlaylists()
|
||||||
|
for key := range ini.Lists {
|
||||||
|
if _, ok := cachedLists[key]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
lists = append(lists, ini.Lists[key])
|
||||||
|
}
|
||||||
|
log.Printf("Found %d cached playlists\n", len(cachedLists))
|
||||||
|
} else {
|
||||||
|
lists = slices.Collect(maps.Values(ini.Lists))
|
||||||
|
}
|
||||||
|
|
||||||
if int(app.Args.RandomCount) > 0 && int(app.Args.RandomCount) <= len(lists) {
|
if int(app.Args.RandomCount) > 0 && int(app.Args.RandomCount) <= len(lists) {
|
||||||
rand.Shuffle(len(lists), func(i int, j int) { lists[i], lists[j] = lists[j], lists[i] })
|
rand.Shuffle(len(lists), func(i int, j int) { lists[i], lists[j] = lists[j], lists[i] })
|
||||||
lists = lists[:app.Args.RandomCount]
|
lists = lists[:app.Args.RandomCount]
|
||||||
@@ -79,17 +96,31 @@ func PrepareListsToCheck(files []string, urls []string, codes []string) []playli
|
|||||||
return lists
|
return lists
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getCachedPlaylists возвращает из кеша проверенные ранее плейлисты
|
||||||
|
func getCachedPlaylists() map[string]playlist.Playlist {
|
||||||
|
result := make(map[string]playlist.Playlist)
|
||||||
|
keys := app.Cache.Keys(ctx, "*")
|
||||||
|
for _, key := range keys.Val() {
|
||||||
|
value := app.Cache.Get(ctx, key).Val()
|
||||||
|
var pls playlist.Playlist
|
||||||
|
_ = json.Unmarshal([]byte(value), &pls)
|
||||||
|
result[pls.Code] = pls
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
// CheckPlaylists проверяет плейлисты и возвращает их же с результатами проверки
|
// CheckPlaylists проверяет плейлисты и возвращает их же с результатами проверки
|
||||||
func CheckPlaylists(lists []playlist.Playlist) (int, int) {
|
func CheckPlaylists(lists []playlist.Playlist) (int, int) {
|
||||||
step, onlineCount, offlineCount := 0, 0, 0
|
|
||||||
count := len(lists)
|
count := len(lists)
|
||||||
tagBlocks = tagfile.Init(app.Args.TagsPath)
|
|
||||||
|
|
||||||
if count == 0 {
|
if count == 0 {
|
||||||
log.Println("There are no playlists to check")
|
log.Println("There are no playlists to check")
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Printf("%d playlists will be checked\n", len(lists))
|
||||||
|
step, onlineCount, offlineCount := 0, 0, 0
|
||||||
|
tagBlocks = tagfile.Init(app.Args.TagsPath)
|
||||||
|
|
||||||
for idx := range lists {
|
for idx := range lists {
|
||||||
pls := lists[idx]
|
pls := lists[idx]
|
||||||
step++
|
step++
|
||||||
@@ -127,6 +158,21 @@ func CheckPlaylists(lists []playlist.Playlist) (int, int) {
|
|||||||
pls = CheckChannels(pls)
|
pls = CheckChannels(pls)
|
||||||
|
|
||||||
lists[idx] = pls
|
lists[idx] = pls
|
||||||
|
|
||||||
|
if app.Config.Cache.IsActive {
|
||||||
|
jsonBytes, err := json.Marshal(pls)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Error while saving playlist to cache: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ttl := time.Duration(app.Config.Cache.Ttl) * time.Second
|
||||||
|
written := app.Cache.Set(ctx, pls.Code, string(jsonBytes), ttl)
|
||||||
|
if written.Err() != nil {
|
||||||
|
log.Printf("Error while saving playlist to cache: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Println("Cached sucessfully")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return onlineCount, offlineCount
|
return onlineCount, offlineCount
|
||||||
@@ -188,7 +234,7 @@ func CheckChannels(pls playlist.Playlist) playlist.Playlist {
|
|||||||
tvChannel.ContentType = resp.Header.Get("Content-Type")
|
tvChannel.ContentType = resp.Header.Get("Content-Type")
|
||||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||||
bodyString := string(bodyBytes)
|
bodyString := string(bodyBytes)
|
||||||
resp.Body.Close()
|
_ = resp.Body.Close()
|
||||||
contentType := http.DetectContentType(bodyBytes)
|
contentType := http.DetectContentType(bodyBytes)
|
||||||
|
|
||||||
isContentBinary := strings.Contains(contentType, "octet-stream") ||
|
isContentBinary := strings.Contains(contentType, "octet-stream") ||
|
||||||
|
|||||||
@@ -7,6 +7,7 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"github.com/joho/godotenv"
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
)
|
)
|
||||||
@@ -14,17 +15,20 @@ import (
|
|||||||
// Config описывает конфигурацию
|
// Config описывает конфигурацию
|
||||||
type Config struct {
|
type Config struct {
|
||||||
DebugMode bool
|
DebugMode bool
|
||||||
Redis RedisConfig
|
Cache CacheConfig
|
||||||
Http HttpConfig
|
Http HttpConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
// RedisConfig описывает конфигурацию подключения к Redis
|
// CacheConfig описывает конфигурацию подключения к keydb
|
||||||
type RedisConfig struct {
|
type CacheConfig struct {
|
||||||
Host string
|
IsEnabled bool
|
||||||
Port uint
|
Host string
|
||||||
Username string
|
Port uint
|
||||||
Password string
|
Username string
|
||||||
Db uint
|
Password string
|
||||||
|
Db uint
|
||||||
|
Ttl uint
|
||||||
|
IsActive bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// HttpConfig описывает конфигурацию веб-сервера
|
// HttpConfig описывает конфигурацию веб-сервера
|
||||||
@@ -35,14 +39,17 @@ type HttpConfig struct {
|
|||||||
|
|
||||||
// Init инициализирует объект конфигурации из переменных окружения
|
// Init инициализирует объект конфигурации из переменных окружения
|
||||||
func Init() *Config {
|
func Init() *Config {
|
||||||
|
_ = godotenv.Load(".env")
|
||||||
return &Config{
|
return &Config{
|
||||||
DebugMode: readEnvBoolean("APP_DEBUG", false),
|
//DebugMode: readEnvBoolean("APP_DEBUG", false),
|
||||||
Redis: RedisConfig{
|
Cache: CacheConfig{
|
||||||
Host: readEnv("REDIS_HOST", ""),
|
IsEnabled: readEnvBoolean("CACHE_ENABLED", false),
|
||||||
Port: readEnvInteger("REDIS_PORT", 6379),
|
Host: readEnv("CACHE_HOST", "localhost"),
|
||||||
Username: readEnv("REDIS_USERNAME", ""),
|
Port: readEnvInteger("CACHE_PORT", 6379),
|
||||||
Password: readEnv("REDIS_PASSWORD", ""),
|
Username: readEnv("CACHE_USERNAME", ""),
|
||||||
Db: readEnvInteger("REDIS_DB", 0),
|
Password: readEnv("CACHE_PASSWORD", ""),
|
||||||
|
Db: readEnvInteger("CACHE_DB", 0),
|
||||||
|
Ttl: readEnvInteger("CACHE_TTL", 1800),
|
||||||
},
|
},
|
||||||
Http: HttpConfig{
|
Http: HttpConfig{
|
||||||
Host: readEnv("HTTP_HOST", "0.0.0.0"),
|
Host: readEnv("HTTP_HOST", "0.0.0.0"),
|
||||||
@@ -57,7 +64,6 @@ func readEnv(key string, defaultValue string) string {
|
|||||||
if exists {
|
if exists {
|
||||||
return value
|
return value
|
||||||
}
|
}
|
||||||
|
|
||||||
return defaultValue
|
return defaultValue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ func Init(path string) (IniFile, error) {
|
|||||||
|
|
||||||
log.Println("Loading playlists from ini-file:", pathNormalized)
|
log.Println("Loading playlists from ini-file:", pathNormalized)
|
||||||
for _, section := range iniFile.Sections() {
|
for _, section := range iniFile.Sections() {
|
||||||
if section.Name() == ini.DefaultSection { //TODO выкосить костыль
|
if section.Name() == ini.DefaultSection {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ package cmd
|
|||||||
import (
|
import (
|
||||||
"axenov/iptv-checker/app"
|
"axenov/iptv-checker/app"
|
||||||
"axenov/iptv-checker/app/checker"
|
"axenov/iptv-checker/app/checker"
|
||||||
"axenov/iptv-checker/app/logger"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
@@ -22,7 +21,7 @@ var checkCmd = &cobra.Command{
|
|||||||
Use: "check",
|
Use: "check",
|
||||||
Short: "Check playlists",
|
Short: "Check playlists",
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
logger.Init(app.Args.NeedQuiet)
|
app.Init()
|
||||||
|
|
||||||
files, _ := cmd.Flags().GetStringSlice("file")
|
files, _ := cmd.Flags().GetStringSlice("file")
|
||||||
urls, _ := cmd.Flags().GetStringSlice("url")
|
urls, _ := cmd.Flags().GetStringSlice("url")
|
||||||
|
|||||||
@@ -27,7 +27,6 @@ Copyright (c) 2025, Антон Аксенов, MIT license.`,
|
|||||||
// Execute adds all child commands to the root command and sets flags appropriately.
|
// Execute adds all child commands to the root command and sets flags appropriately.
|
||||||
// This is called by main.main(). It only needs to happen once to the rootCmd.
|
// This is called by main.main(). It only needs to happen once to the rootCmd.
|
||||||
func Execute() {
|
func Execute() {
|
||||||
app.Init()
|
|
||||||
err := rootCmd.Execute()
|
err := rootCmd.Execute()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
|
|||||||
3
go.mod
3
go.mod
@@ -3,7 +3,9 @@ module axenov/iptv-checker
|
|||||||
go 1.23.6
|
go 1.23.6
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/joho/godotenv v1.5.1
|
||||||
github.com/redis/go-redis/v9 v9.7.3
|
github.com/redis/go-redis/v9 v9.7.3
|
||||||
|
github.com/spf13/cobra v1.9.1
|
||||||
gopkg.in/ini.v1 v1.67.0
|
gopkg.in/ini.v1 v1.67.0
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -12,7 +14,6 @@ require (
|
|||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
github.com/spf13/cobra v1.9.1 // indirect
|
|
||||||
github.com/spf13/pflag v1.0.6 // indirect
|
github.com/spf13/pflag v1.0.6 // indirect
|
||||||
github.com/stretchr/testify v1.7.0 // indirect
|
github.com/stretchr/testify v1.7.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
4
go.sum
4
go.sum
@@ -12,6 +12,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
|
|||||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
|
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||||
|
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
|
github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
|
||||||
@@ -27,6 +29,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
|||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
Reference in New Issue
Block a user