Реализовано кеширование проверенных плейлистов, при включенном кеше -r теперь не учитывает только некешированные из ini-файла

2025-05-08 11:14:59 +08:00
parent 994df87846
commit c1a7f7e289
10 changed files with 106 additions and 47 deletions
--- a/.env.example
+++ b/.env.example
@@ -3,8 +3,10 @@ APP_DEBUG=false
 HTTP_HOST=0.0.0.0
 HTTP_PORT=8031

-REDIS_HOST=
-REDIS_PORT=
-REDIS_USERNAME=
-REDIS_PASSWORD=
-REDIS_DB=
+CACHE_ENABLED=false
+CACHE_HOST=localhost
+CACHE_PORT=6379
+CACHE_USERNAME=
+CACHE_PASSWORD=
+CACHE_DB=1
+CACHE_TTL=1800
--- a/app/app.go
+++ b/app/app.go
@@ -7,7 +7,9 @@
 package app

 import (
+	"axenov/iptv-checker/app/cache"
 	"axenov/iptv-checker/app/config"
+	"axenov/iptv-checker/app/logger"
 	"github.com/redis/go-redis/v9"
 )

@@ -25,14 +27,15 @@ type Arguments struct {

 var (
 	Args   Arguments
-	Redis  *redis.Client
+	Cache  *redis.Client
 	Config *config.Config
-	//TagBlocks []tagfile.TagBlock
 )

-// Init инициализирует глобальные переменные
+// Init инициализирует конфигурацию и подключение к keydb
 func Init() {
 	Config = config.Init()
-	//logger.Init(Args.NeedQuiet)
-	//Redis = cache.Init(Config.Redis)
+	logger.Init(Args.NeedQuiet)
+	if Config.Cache.IsEnabled {
+		Cache = cache.Init(&Config.Cache)
+	}
 }
--- a/app/cache/cache.go
+++ b/app/cache/cache.go
@@ -15,22 +15,23 @@ import (
 	"strconv"
 )

-func Init(cfg config.RedisConfig) *redis.Client {
-	rdb := redis.NewClient(&redis.Options{
+func Init(cfg *config.CacheConfig) *redis.Client {
+	redis := redis.NewClient(&redis.Options{
 		Addr:         fmt.Sprintf("%s:%s", cfg.Host, strconv.Itoa(int(cfg.Port))),
 		DB:           int(cfg.Db),
 		PoolSize:     1000,
 		ReadTimeout:  -1,
 		WriteTimeout: -1,
 	})
-	client := rdb.Conn()
-
-	var ctx context.Context
-	if client.Ping(ctx).Err() != nil {
-		log.Println("Error while connecting to Redis", cfg.Host, cfg.Port, cfg.Db)
+	client := redis.Conn()
+	ctx := context.Background()
+	err := client.Ping(ctx).Err()
+	if err == nil {
+		log.Println("Connected to cache DB")
+		cfg.IsActive = true
 	} else {
-		log.Println("Connected to Redis", cfg.Host, cfg.Port, cfg.Db)
+		log.Println("Error while connecting to cache DB, program may work not as expected:", err)
 	}

-	return rdb
+	return redis
 }
--- a/app/checker/checker.go
+++ b/app/checker/checker.go
@@ -12,7 +12,9 @@ import (
 	"axenov/iptv-checker/app/playlist"
 	"axenov/iptv-checker/app/tagfile"
 	"axenov/iptv-checker/app/utils"
+	"context"
 	"crypto/tls"
+	"encoding/json"
 	"io"
 	"log"
 	"maps"
@@ -26,7 +28,10 @@ import (
 	"time"
 )

-var tagBlocks []tagfile.TagBlock
+var (
+	tagBlocks []tagfile.TagBlock
+	ctx       = context.Background()
+)

 // PrepareListsToCheck готовит список плейлистов для проверки
 func PrepareListsToCheck(files []string, urls []string, codes []string) []playlist.Playlist {
@@ -68,7 +73,19 @@ func PrepareListsToCheck(files []string, urls []string, codes []string) []playli
 				lists = append(lists, list)
 			}
 		} else {
-			lists = slices.Collect(maps.Values(ini.Lists))
+			if app.Config.Cache.IsActive {
+				cachedLists := getCachedPlaylists()
+				for key := range ini.Lists {
+					if _, ok := cachedLists[key]; ok {
+						continue
+					}
+					lists = append(lists, ini.Lists[key])
+				}
+				log.Printf("Found %d cached playlists\n", len(cachedLists))
+			} else {
+				lists = slices.Collect(maps.Values(ini.Lists))
+			}
+
 			if int(app.Args.RandomCount) > 0 && int(app.Args.RandomCount) <= len(lists) {
 				rand.Shuffle(len(lists), func(i int, j int) { lists[i], lists[j] = lists[j], lists[i] })
 				lists = lists[:app.Args.RandomCount]
@@ -79,17 +96,31 @@ func PrepareListsToCheck(files []string, urls []string, codes []string) []playli
 	return lists
 }

+// getCachedPlaylists возвращает из кеша проверенные ранее плейлисты
+func getCachedPlaylists() map[string]playlist.Playlist {
+	result := make(map[string]playlist.Playlist)
+	keys := app.Cache.Keys(ctx, "*")
+	for _, key := range keys.Val() {
+		value := app.Cache.Get(ctx, key).Val()
+		var pls playlist.Playlist
+		_ = json.Unmarshal([]byte(value), &pls)
+		result[pls.Code] = pls
+	}
+	return result
+}
+
 // CheckPlaylists проверяет плейлисты и возвращает их же с результатами проверки
 func CheckPlaylists(lists []playlist.Playlist) (int, int) {
-	step, onlineCount, offlineCount := 0, 0, 0
 	count := len(lists)
-	tagBlocks = tagfile.Init(app.Args.TagsPath)
-
 	if count == 0 {
 		log.Println("There are no playlists to check")
 		os.Exit(0)
 	}

+	log.Printf("%d playlists will be checked\n", len(lists))
+	step, onlineCount, offlineCount := 0, 0, 0
+	tagBlocks = tagfile.Init(app.Args.TagsPath)
+
 	for idx := range lists {
 		pls := lists[idx]
 		step++
@@ -127,6 +158,21 @@ func CheckPlaylists(lists []playlist.Playlist) (int, int) {
 		pls = CheckChannels(pls)

 		lists[idx] = pls
+
+		if app.Config.Cache.IsActive {
+			jsonBytes, err := json.Marshal(pls)
+			if err != nil {
+				log.Printf("Error while saving playlist to cache: %s", err)
+			}
+
+			ttl := time.Duration(app.Config.Cache.Ttl) * time.Second
+			written := app.Cache.Set(ctx, pls.Code, string(jsonBytes), ttl)
+			if written.Err() != nil {
+				log.Printf("Error while saving playlist to cache: %s", err)
+			}
+
+			log.Println("Cached sucessfully")
+		}
 	}

 	return onlineCount, offlineCount
@@ -188,7 +234,7 @@ func CheckChannels(pls playlist.Playlist) playlist.Playlist {
 			tvChannel.ContentType = resp.Header.Get("Content-Type")
 			bodyBytes, _ := io.ReadAll(resp.Body)
 			bodyString := string(bodyBytes)
-			resp.Body.Close()
+			_ = resp.Body.Close()
 			contentType := http.DetectContentType(bodyBytes)

 			isContentBinary := strings.Contains(contentType, "octet-stream") ||
--- a/app/config/config.go
+++ b/app/config/config.go
@@ -7,6 +7,7 @@
 package config

 import (
+	"github.com/joho/godotenv"
 	"os"
 	"strconv"
 )
@@ -14,17 +15,20 @@ import (
 // Config описывает конфигурацию
 type Config struct {
 	DebugMode bool
-	Redis     RedisConfig
+	Cache     CacheConfig
 	Http      HttpConfig
 }

-// RedisConfig описывает конфигурацию подключения к Redis
-type RedisConfig struct {
-	Host     string
-	Port     uint
-	Username string
-	Password string
-	Db       uint
+// CacheConfig описывает конфигурацию подключения к keydb
+type CacheConfig struct {
+	IsEnabled bool
+	Host      string
+	Port      uint
+	Username  string
+	Password  string
+	Db        uint
+	Ttl       uint
+	IsActive  bool
 }

 // HttpConfig описывает конфигурацию веб-сервера
@@ -35,14 +39,17 @@ type HttpConfig struct {

 // Init инициализирует объект конфигурации из переменных окружения
 func Init() *Config {
+	_ = godotenv.Load(".env")
 	return &Config{
-		DebugMode: readEnvBoolean("APP_DEBUG", false),
-		Redis: RedisConfig{
-			Host:     readEnv("REDIS_HOST", ""),
-			Port:     readEnvInteger("REDIS_PORT", 6379),
-			Username: readEnv("REDIS_USERNAME", ""),
-			Password: readEnv("REDIS_PASSWORD", ""),
-			Db:       readEnvInteger("REDIS_DB", 0),
+		//DebugMode: readEnvBoolean("APP_DEBUG", false),
+		Cache: CacheConfig{
+			IsEnabled: readEnvBoolean("CACHE_ENABLED", false),
+			Host:      readEnv("CACHE_HOST", "localhost"),
+			Port:      readEnvInteger("CACHE_PORT", 6379),
+			Username:  readEnv("CACHE_USERNAME", ""),
+			Password:  readEnv("CACHE_PASSWORD", ""),
+			Db:        readEnvInteger("CACHE_DB", 0),
+			Ttl:       readEnvInteger("CACHE_TTL", 1800),
 		},
 		Http: HttpConfig{
 			Host: readEnv("HTTP_HOST", "0.0.0.0"),
@@ -57,7 +64,6 @@ func readEnv(key string, defaultValue string) string {
 	if exists {
 		return value
 	}
-
 	return defaultValue
 }

--- a/app/inifile/inifile.go
+++ b/app/inifile/inifile.go
@@ -44,7 +44,7 @@ func Init(path string) (IniFile, error) {

 	log.Println("Loading playlists from ini-file:", pathNormalized)
 	for _, section := range iniFile.Sections() {
-		if section.Name() == ini.DefaultSection { //TODO выкосить костыль
+		if section.Name() == ini.DefaultSection {
 			continue
 		}

--- a/cmd/check.go
+++ b/cmd/check.go
@@ -9,7 +9,6 @@ package cmd
 import (
 	"axenov/iptv-checker/app"
 	"axenov/iptv-checker/app/checker"
-	"axenov/iptv-checker/app/logger"
 	"encoding/json"
 	"fmt"
 	"github.com/spf13/cobra"
@@ -22,7 +21,7 @@ var checkCmd = &cobra.Command{
 	Use:   "check",
 	Short: "Check playlists",
 	Run: func(cmd *cobra.Command, args []string) {
-		logger.Init(app.Args.NeedQuiet)
+		app.Init()

 		files, _ := cmd.Flags().GetStringSlice("file")
 		urls, _ := cmd.Flags().GetStringSlice("url")
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -27,7 +27,6 @@ Copyright (c) 2025, Антон Аксенов, MIT license.`,
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
-	app.Init()
 	err := rootCmd.Execute()
 	if err != nil {
 		os.Exit(1)
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,9 @@ module axenov/iptv-checker
 go 1.23.6

 require (
+	github.com/joho/godotenv v1.5.1
 	github.com/redis/go-redis/v9 v9.7.3
+	github.com/spf13/cobra v1.9.1
 	gopkg.in/ini.v1 v1.67.0
 )

@@ -12,7 +14,6 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/spf13/cobra v1.9.1 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -12,6 +12,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
@@ -27,6 +29,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=