From c1a7f7e289cd9f187a52174839230aaea6bea6cc Mon Sep 17 00:00:00 2001 From: AnthonyAxenov Date: Thu, 8 May 2025 11:14:59 +0800 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D0=BD=D0=BE=20=D0=BA=D0=B5=D1=88=D0=B8=D1=80=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5?= =?UTF-8?q?=D1=80=D0=B5=D0=BD=D0=BD=D1=8B=D1=85=20=D0=BF=D0=BB=D0=B5=D0=B9?= =?UTF-8?q?=D0=BB=D0=B8=D1=81=D1=82=D0=BE=D0=B2,=20=D0=BF=D1=80=D0=B8=20?= =?UTF-8?q?=D0=B2=D0=BA=D0=BB=D1=8E=D1=87=D0=B5=D0=BD=D0=BD=D0=BE=D0=BC=20?= =?UTF-8?q?=D0=BA=D0=B5=D1=88=D0=B5=20-r=20=D1=82=D0=B5=D0=BF=D0=B5=D1=80?= =?UTF-8?q?=D1=8C=20=D0=BD=D0=B5=20=D1=83=D1=87=D0=B8=D1=82=D1=8B=D0=B2?= =?UTF-8?q?=D0=B0=D0=B5=D1=82=20=D1=82=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE=20?= =?UTF-8?q?=D0=BD=D0=B5=D0=BA=D0=B5=D1=88=D0=B8=D1=80=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D0=BD=D0=BD=D1=8B=D0=B5=20=D0=B8=D0=B7=20ini-=D1=84=D0=B0?= =?UTF-8?q?=D0=B9=D0=BB=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 12 +++++---- app/app.go | 13 ++++++---- app/cache/cache.go | 19 +++++++------- app/checker/checker.go | 58 +++++++++++++++++++++++++++++++++++++----- app/config/config.go | 38 +++++++++++++++------------ app/inifile/inifile.go | 2 +- cmd/check.go | 3 +-- cmd/root.go | 1 - go.mod | 3 ++- go.sum | 4 ++- 10 files changed, 106 insertions(+), 47 deletions(-) diff --git a/.env.example b/.env.example index d88c416..514e72a 100644 --- a/.env.example +++ b/.env.example @@ -3,8 +3,10 @@ APP_DEBUG=false HTTP_HOST=0.0.0.0 HTTP_PORT=8031 -REDIS_HOST= -REDIS_PORT= -REDIS_USERNAME= -REDIS_PASSWORD= -REDIS_DB= +CACHE_ENABLED=false +CACHE_HOST=localhost +CACHE_PORT=6379 +CACHE_USERNAME= +CACHE_PASSWORD= +CACHE_DB=1 +CACHE_TTL=1800 diff --git a/app/app.go b/app/app.go index 9fdca57..243d4b7 100644 --- a/app/app.go +++ b/app/app.go @@ -7,7 +7,9 @@ package app import ( + "axenov/iptv-checker/app/cache" "axenov/iptv-checker/app/config" + "axenov/iptv-checker/app/logger" "github.com/redis/go-redis/v9" ) @@ -25,14 +27,15 @@ type Arguments struct { var ( Args Arguments - Redis *redis.Client + Cache *redis.Client Config *config.Config - //TagBlocks []tagfile.TagBlock ) -// Init инициализирует глобальные переменные +// Init инициализирует конфигурацию и подключение к keydb func Init() { Config = config.Init() - //logger.Init(Args.NeedQuiet) - //Redis = cache.Init(Config.Redis) + logger.Init(Args.NeedQuiet) + if Config.Cache.IsEnabled { + Cache = cache.Init(&Config.Cache) + } } diff --git a/app/cache/cache.go b/app/cache/cache.go index 0b9f931..9e9a623 100644 --- a/app/cache/cache.go +++ b/app/cache/cache.go @@ -15,22 +15,23 @@ import ( "strconv" ) -func Init(cfg config.RedisConfig) *redis.Client { - rdb := redis.NewClient(&redis.Options{ +func Init(cfg *config.CacheConfig) *redis.Client { + redis := redis.NewClient(&redis.Options{ Addr: fmt.Sprintf("%s:%s", cfg.Host, strconv.Itoa(int(cfg.Port))), DB: int(cfg.Db), PoolSize: 1000, ReadTimeout: -1, WriteTimeout: -1, }) - client := rdb.Conn() - - var ctx context.Context - if client.Ping(ctx).Err() != nil { - log.Println("Error while connecting to Redis", cfg.Host, cfg.Port, cfg.Db) + client := redis.Conn() + ctx := context.Background() + err := client.Ping(ctx).Err() + if err == nil { + log.Println("Connected to cache DB") + cfg.IsActive = true } else { - log.Println("Connected to Redis", cfg.Host, cfg.Port, cfg.Db) + log.Println("Error while connecting to cache DB, program may work not as expected:", err) } - return rdb + return redis } diff --git a/app/checker/checker.go b/app/checker/checker.go index 021e5b7..c216056 100644 --- a/app/checker/checker.go +++ b/app/checker/checker.go @@ -12,7 +12,9 @@ import ( "axenov/iptv-checker/app/playlist" "axenov/iptv-checker/app/tagfile" "axenov/iptv-checker/app/utils" + "context" "crypto/tls" + "encoding/json" "io" "log" "maps" @@ -26,7 +28,10 @@ import ( "time" ) -var tagBlocks []tagfile.TagBlock +var ( + tagBlocks []tagfile.TagBlock + ctx = context.Background() +) // PrepareListsToCheck готовит список плейлистов для проверки func PrepareListsToCheck(files []string, urls []string, codes []string) []playlist.Playlist { @@ -68,7 +73,19 @@ func PrepareListsToCheck(files []string, urls []string, codes []string) []playli lists = append(lists, list) } } else { - lists = slices.Collect(maps.Values(ini.Lists)) + if app.Config.Cache.IsActive { + cachedLists := getCachedPlaylists() + for key := range ini.Lists { + if _, ok := cachedLists[key]; ok { + continue + } + lists = append(lists, ini.Lists[key]) + } + log.Printf("Found %d cached playlists\n", len(cachedLists)) + } else { + lists = slices.Collect(maps.Values(ini.Lists)) + } + if int(app.Args.RandomCount) > 0 && int(app.Args.RandomCount) <= len(lists) { rand.Shuffle(len(lists), func(i int, j int) { lists[i], lists[j] = lists[j], lists[i] }) lists = lists[:app.Args.RandomCount] @@ -79,17 +96,31 @@ func PrepareListsToCheck(files []string, urls []string, codes []string) []playli return lists } +// getCachedPlaylists возвращает из кеша проверенные ранее плейлисты +func getCachedPlaylists() map[string]playlist.Playlist { + result := make(map[string]playlist.Playlist) + keys := app.Cache.Keys(ctx, "*") + for _, key := range keys.Val() { + value := app.Cache.Get(ctx, key).Val() + var pls playlist.Playlist + _ = json.Unmarshal([]byte(value), &pls) + result[pls.Code] = pls + } + return result +} + // CheckPlaylists проверяет плейлисты и возвращает их же с результатами проверки func CheckPlaylists(lists []playlist.Playlist) (int, int) { - step, onlineCount, offlineCount := 0, 0, 0 count := len(lists) - tagBlocks = tagfile.Init(app.Args.TagsPath) - if count == 0 { log.Println("There are no playlists to check") os.Exit(0) } + log.Printf("%d playlists will be checked\n", len(lists)) + step, onlineCount, offlineCount := 0, 0, 0 + tagBlocks = tagfile.Init(app.Args.TagsPath) + for idx := range lists { pls := lists[idx] step++ @@ -127,6 +158,21 @@ func CheckPlaylists(lists []playlist.Playlist) (int, int) { pls = CheckChannels(pls) lists[idx] = pls + + if app.Config.Cache.IsActive { + jsonBytes, err := json.Marshal(pls) + if err != nil { + log.Printf("Error while saving playlist to cache: %s", err) + } + + ttl := time.Duration(app.Config.Cache.Ttl) * time.Second + written := app.Cache.Set(ctx, pls.Code, string(jsonBytes), ttl) + if written.Err() != nil { + log.Printf("Error while saving playlist to cache: %s", err) + } + + log.Println("Cached sucessfully") + } } return onlineCount, offlineCount @@ -188,7 +234,7 @@ func CheckChannels(pls playlist.Playlist) playlist.Playlist { tvChannel.ContentType = resp.Header.Get("Content-Type") bodyBytes, _ := io.ReadAll(resp.Body) bodyString := string(bodyBytes) - resp.Body.Close() + _ = resp.Body.Close() contentType := http.DetectContentType(bodyBytes) isContentBinary := strings.Contains(contentType, "octet-stream") || diff --git a/app/config/config.go b/app/config/config.go index ed78801..c171af9 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -7,6 +7,7 @@ package config import ( + "github.com/joho/godotenv" "os" "strconv" ) @@ -14,17 +15,20 @@ import ( // Config описывает конфигурацию type Config struct { DebugMode bool - Redis RedisConfig + Cache CacheConfig Http HttpConfig } -// RedisConfig описывает конфигурацию подключения к Redis -type RedisConfig struct { - Host string - Port uint - Username string - Password string - Db uint +// CacheConfig описывает конфигурацию подключения к keydb +type CacheConfig struct { + IsEnabled bool + Host string + Port uint + Username string + Password string + Db uint + Ttl uint + IsActive bool } // HttpConfig описывает конфигурацию веб-сервера @@ -35,14 +39,17 @@ type HttpConfig struct { // Init инициализирует объект конфигурации из переменных окружения func Init() *Config { + _ = godotenv.Load(".env") return &Config{ - DebugMode: readEnvBoolean("APP_DEBUG", false), - Redis: RedisConfig{ - Host: readEnv("REDIS_HOST", ""), - Port: readEnvInteger("REDIS_PORT", 6379), - Username: readEnv("REDIS_USERNAME", ""), - Password: readEnv("REDIS_PASSWORD", ""), - Db: readEnvInteger("REDIS_DB", 0), + //DebugMode: readEnvBoolean("APP_DEBUG", false), + Cache: CacheConfig{ + IsEnabled: readEnvBoolean("CACHE_ENABLED", false), + Host: readEnv("CACHE_HOST", "localhost"), + Port: readEnvInteger("CACHE_PORT", 6379), + Username: readEnv("CACHE_USERNAME", ""), + Password: readEnv("CACHE_PASSWORD", ""), + Db: readEnvInteger("CACHE_DB", 0), + Ttl: readEnvInteger("CACHE_TTL", 1800), }, Http: HttpConfig{ Host: readEnv("HTTP_HOST", "0.0.0.0"), @@ -57,7 +64,6 @@ func readEnv(key string, defaultValue string) string { if exists { return value } - return defaultValue } diff --git a/app/inifile/inifile.go b/app/inifile/inifile.go index cbf5d22..7ce2dd0 100644 --- a/app/inifile/inifile.go +++ b/app/inifile/inifile.go @@ -44,7 +44,7 @@ func Init(path string) (IniFile, error) { log.Println("Loading playlists from ini-file:", pathNormalized) for _, section := range iniFile.Sections() { - if section.Name() == ini.DefaultSection { //TODO выкосить костыль + if section.Name() == ini.DefaultSection { continue } diff --git a/cmd/check.go b/cmd/check.go index f221bcf..1570a5b 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -9,7 +9,6 @@ package cmd import ( "axenov/iptv-checker/app" "axenov/iptv-checker/app/checker" - "axenov/iptv-checker/app/logger" "encoding/json" "fmt" "github.com/spf13/cobra" @@ -22,7 +21,7 @@ var checkCmd = &cobra.Command{ Use: "check", Short: "Check playlists", Run: func(cmd *cobra.Command, args []string) { - logger.Init(app.Args.NeedQuiet) + app.Init() files, _ := cmd.Flags().GetStringSlice("file") urls, _ := cmd.Flags().GetStringSlice("url") diff --git a/cmd/root.go b/cmd/root.go index ce0c032..82b4cef 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,7 +27,6 @@ Copyright (c) 2025, Антон Аксенов, MIT license.`, // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { - app.Init() err := rootCmd.Execute() if err != nil { os.Exit(1) diff --git a/go.mod b/go.mod index 62ada80..7831e3f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,9 @@ module axenov/iptv-checker go 1.23.6 require ( + github.com/joho/godotenv v1.5.1 github.com/redis/go-redis/v9 v9.7.3 + github.com/spf13/cobra v1.9.1 gopkg.in/ini.v1 v1.67.0 ) @@ -12,7 +14,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/spf13/cobra v1.9.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/stretchr/testify v1.7.0 // indirect ) diff --git a/go.sum b/go.sum index dfb2257..0fd51de 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= @@ -27,6 +29,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=