wip8

2026-07-13 12:28:59 +08:00
parent 6c3de4b2ef
commit cd2ab11c44
65 changed files with 9901 additions and 413 deletions
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2025, Антон Аксенов
+ * Copyright (c) 2025-2026, Антон Аксенов
 * This file is part of iptvc project
 * MIT License: https://git.axenov.dev/IPTV/iptvc/src/branch/master/LICENSE
 */
@@ -15,13 +15,12 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/json"
+	"fmt"
 	"io"
 	"log"
 	"maps"
-	"math"
 	"math/rand"
 	"net/http"
-	"os"
 	"slices"
 	"strings"
 	"sync"
@@ -29,8 +28,9 @@ import (
 )

 var (
-	tagBlocks []tagfile.TagBlock
-	ctx       = context.Background()
+	tagBlocks         []tagfile.TagBlock
+	ctx               = context.Background()
+	OnPlaylistChecked func(playlist.Playlist) // вызывается после проверки каждого плейлиста
 )

 // PrepareListsToCheck готовит список плейлистов для проверки
@@ -57,7 +57,7 @@ func PrepareListsToCheck(files []string, urls []string, codes []string) []playli
 	}

 	if len(lists) == 0 || len(codes) > 0 {
-		ini, err := inifile.Init(app.Args.IniPath)
+		ini, err := inifile.Init(app.Config.App.Playlists)
 		if err != nil {
 			log.Printf("Warning: %s, all --code flags will be ignored\n", err)
 			return lists
@@ -109,56 +109,100 @@ func getCachedPlaylists() map[string]playlist.Playlist {
 	return result
 }

-// CheckPlaylists проверяет плейлисты и возвращает их же с результатами проверки
+// CheckPlaylists проверяет плейлисты и возвращает их же с результатами проверки.
+// Параметры (user-agent, timeout, cooldown, per-routine) берутся из config.Check.Playlists.
 func CheckPlaylists(lists []playlist.Playlist) (int, int) {
 	count := len(lists)
 	if count == 0 {
 		log.Println("There are no playlists to check")
-		os.Exit(0)
+		return 0, 0
 	}

-	log.Printf("%d playlists will be checked\n", len(lists))
-	step, onlineCount, offlineCount := 0, 0, 0
-	tagBlocks = tagfile.Init(app.Args.TagsPath)
+	pc := app.Config.Check.Playlists
+	maxRoutines := pc.MaxRoutines
+	if maxRoutines < 1 {
+		maxRoutines = 1
+	}
+
+	log.Printf("%d playlists will be checked (max-routines=%d)\n", count, maxRoutines)
+	tagBlocks = tagfile.Init(app.Config.App.Tags)
+
+	type checkResult struct {
+		idx    int
+		pls    playlist.Playlist
+		online bool
+	}
+
+	sem := make(chan struct{}, maxRoutines)
+	var wg sync.WaitGroup
+	results := make([]checkResult, count)

 	for idx := range lists {
-		pls := lists[idx]
-		step++
+		wg.Add(1)
+		go func(idx int) {
+			sem <- struct{}{}
+			defer func() {
+				<-sem
+				wg.Done()
+			}()

-		var err error
-		if pls.Source == "-f" {
-			// direct m3u path
-			log.Printf("[%.3d/%.3d] Playlist from filesystem\n", step, count)
-			log.Printf("Reading file... (%s)\n", pls.Url)
-			err = pls.ReadFromFs()
-		} else if pls.Source == "-u" {
-			// direct m3u url
-			log.Printf("[%.3d/%.3d] Playlist [%s]\n", step, count, pls.Url)
-			log.Printf("Fetching... (%s)\n", pls.Url)
-			err = pls.Download()
-		} else {
-			// from ini
-			log.Printf("[%.3d/%.3d] Playlist [%s]\n", step, count, pls.Code)
-			log.Printf("Fetching... (%s)\n", pls.Url)
-			err = pls.Download()
-		}
+			pls := lists[idx]
+			pls.CheckedAt = time.Now().Unix()

-		if err != nil {
-			log.Printf("Cannot read playlist [%s]: %s\n", pls.Url, err)
-			offlineCount++
+			userAgent := pc.UserAgent.Pick()
+			timeout := time.Duration(pc.Timeout) * time.Millisecond
+
+			var err error
+			if pls.Source == "-f" {
+				log.Printf("Playlist from filesystem (%s)\n", pls.Url)
+				err = pls.ReadFromFs()
+			} else {
+				log.Printf("Fetching playlist [%s] (%s)\n", pls.Code, pls.Url)
+				err = pls.Download(userAgent, timeout)
+			}
+
+			if err != nil {
+				log.Printf("Cannot read playlist [%s]: %s\n", pls.Url, err)
+				pls.IsOnline = false
+				results[idx] = checkResult{idx: idx, pls: pls, online: false}
+				cachePlaylist(pls)
+				if OnPlaylistChecked != nil {
+					OnPlaylistChecked(pls)
+				}
+				return
+			}
+
+			log.Println("Parsing content...")
+			pls.IsOnline = true
+			pls = pls.Parse()
+
+			log.Printf("Parsed, checking channels (%d)...\n", len(pls.Channels))
+			pls = CheckChannels(pls)
+			results[idx] = checkResult{idx: idx, pls: pls, online: true}
 			cachePlaylist(pls)
-			continue
+			if OnPlaylistChecked != nil {
+				OnPlaylistChecked(pls)
+			}
+
+			// one-cooldown: задержка после проверки каждого плейлиста
+			// pc.OneCooldown хранится в миллисекундах, поэтому переводим в time.Millisecond
+			oneCd := time.Duration(pc.OneCooldown.Value()) * time.Millisecond
+			if oneCd > 0 {
+				time.Sleep(oneCd)
+			}
+		}(idx)
+	}
+
+	wg.Wait()
+
+	onlineCount, offlineCount := 0, 0
+	for _, r := range results {
+		lists[r.idx] = r.pls
+		if r.online {
+			onlineCount++
+		} else {
+			offlineCount++
 		}
-
-		log.Println("Parsing content...")
-		pls.IsOnline = true
-		onlineCount++
-		pls = pls.Parse()
-
-		log.Printf("Parsed, checking channels (%d)...\n", len(pls.Channels))
-		pls = CheckChannels(pls)
-		lists[idx] = pls
-		cachePlaylist(pls)
 	}

 	return onlineCount, offlineCount
@@ -183,7 +227,8 @@ func cachePlaylist(pls playlist.Playlist) {
 	log.Println("Cached sucessfully")
 }

-// CheckChannels проверяет каналы и возвращает их же с результатами проверки
+// CheckChannels проверяет каналы и возвращает их же с результатами проверки.
+// Параметры (user-agent, timeout, byte-range, cooldown, per-routine) берутся из config.Check.Channels.
 func CheckChannels(pls playlist.Playlist) playlist.Playlist {
 	type errorData struct {
 		tvChannel playlist.Channel
@@ -199,14 +244,27 @@ func CheckChannels(pls playlist.Playlist) playlist.Playlist {
 	pls.OnlineCount = 0
 	pls.OfflineCount = 0

-	timeout, routines := calcParameters(count)
+	cc := app.Config.Check.Channels
+	timeoutMs := cc.Timeout
+	byteRange := cc.ByteRange
+	if byteRange <= 0 {
+		byteRange = 512
+	}
+	maxRoutines := cc.MaxRoutines
+	if maxRoutines < 1 {
+		maxRoutines = 1
+	}
+
+	timeout := time.Duration(timeoutMs) * time.Millisecond
 	httpClient := http.Client{Timeout: timeout}
-	chSemaphores := make(chan struct{}, routines)
+	chSemaphores := make(chan struct{}, maxRoutines)
 	chOnline := make(chan playlist.Channel, len(pls.Channels))
 	chOffline := make(chan playlist.Channel, len(pls.Channels))
 	chError := make(chan errorData, len(pls.Channels))
 	var wg sync.WaitGroup

+	log.Printf("Check channels parameters: timeout=%.2fs byte-range=%d max-routines=%d\n", float64(timeoutMs)/1000, byteRange, maxRoutines)
+
 	startTime := time.Now()
 	for _, tvChannel := range pls.Channels {
 		wg.Add(1)
@@ -227,8 +285,8 @@ func CheckChannels(pls playlist.Playlist) playlist.Playlist {
 				return
 			}

-			req.Header.Set("User-Agent", "Mozilla/5.0 WINK/1.31.1 (AndroidTV/9) HlsWinkPlayer")
-			req.Header.Set("Range", "bytes=0-511") // 512 B, but sometimes servers ignore it
+			req.Header.Set("User-Agent", cc.UserAgent.Pick())
+			req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", byteRange-1))
 			resp, err := httpClient.Do(req)
 			tvChannel.CheckedAt = time.Now().Unix()
 			if err != nil {
@@ -240,7 +298,7 @@ func CheckChannels(pls playlist.Playlist) playlist.Playlist {
 			tvChannel.Status = resp.StatusCode
 			tvChannel.IsOnline = tvChannel.Status < http.StatusBadRequest
 			tvChannel.ContentType = resp.Header.Get("Content-Type")
-			chunk := io.LimitReader(resp.Body, 512) // just for sure
+			chunk := io.LimitReader(resp.Body, int64(byteRange))
 			bodyBytes, _ := io.ReadAll(chunk)
 			bodyString := string(bodyBytes)
 			_ = resp.Body.Close()
@@ -263,7 +321,13 @@ func CheckChannels(pls playlist.Playlist) playlist.Playlist {
 			}

 			chOnline <- tvChannel
-			return
+
+			// cooldown: задержка после проверки каждого канала
+			// cc.Cooldown хранится в миллисекундах, поэтому переводим в time.Millisecond
+			cd := time.Duration(cc.Cooldown.Value()) * time.Millisecond
+			if cd > 0 {
+				time.Sleep(cd)
+			}
 		}(tvChannel)
 	}

@@ -323,42 +387,6 @@ func CheckChannels(pls playlist.Playlist) playlist.Playlist {
 	return pls
 }

-// calcParameters вычисляет оптимальное количество горутин и таймаут запроса
-func calcParameters(count int) (time.Duration, int) {
-	routines := count
-	if routines > 3000 {
-		routines = 3000
-	}
-	if routines < 1 {
-		routines = 1
-	}
-
-	var digits = 1
-	x := count
-	for x >= 10 {
-		digits++
-		x /= 10
-	}
-
-	timeout := 10 - int(math.Ceil(float64(digits)*1.5))
-	if timeout > 10 {
-		timeout = 10
-	}
-	if timeout < 1 {
-		timeout = 1
-	}
-
-	duration := time.Duration(timeout) * time.Second
-	log.Printf(
-		"Check parameters calculated count=%d timeout=%.2fs routines=%d\n",
-		count,
-		duration.Seconds(),
-		routines,
-	)
-
-	return duration, routines
-}
-
 // getTagsForChannel ищет и возвращает теги для канала
 func getTagsForChannel(tvChannel playlist.Channel) []string {
 	var foundTags []string