427 lines
11 KiB
Go
427 lines
11 KiB
Go
/*
|
||
* Copyright (c) 2025, Антон Аксенов
|
||
* This file is part of iptvc project
|
||
* MIT License: https://git.axenov.dev/IPTV/iptvc/src/branch/master/LICENSE
|
||
*/
|
||
|
||
package checker
|
||
|
||
import (
|
||
"axenov/iptv-checker/app"
|
||
"axenov/iptv-checker/app/inifile"
|
||
"axenov/iptv-checker/app/playlist"
|
||
"axenov/iptv-checker/app/tagfile"
|
||
"axenov/iptv-checker/app/utils"
|
||
"context"
|
||
"crypto/tls"
|
||
"encoding/json"
|
||
"io"
|
||
"log"
|
||
"maps"
|
||
"math"
|
||
"math/rand"
|
||
"net/http"
|
||
"os"
|
||
"slices"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
// Checker выполняет проверку плейлистов и каналов.
|
||
type Checker struct {
|
||
tagBlocks []tagfile.TagBlock
|
||
}
|
||
|
||
// NewChecker создаёт новый экземпляр Checker.
|
||
func NewChecker(tagsPath string) *Checker {
|
||
return &Checker{
|
||
tagBlocks: tagfile.Init(tagsPath),
|
||
}
|
||
}
|
||
|
||
// PrepareListsToCheck готовит список плейлистов для проверки
|
||
func (c *Checker) PrepareListsToCheck(files []string, urls []string, codes []string) []playlist.Playlist {
|
||
var lists []playlist.Playlist
|
||
|
||
if len(files) > 0 {
|
||
for _, filepath := range files {
|
||
pls, err := playlist.MakeFromFile(filepath)
|
||
if err != nil {
|
||
log.Printf("Warning: %s, skipping\n", err)
|
||
continue
|
||
}
|
||
|
||
lists = append(lists, pls)
|
||
}
|
||
}
|
||
|
||
if len(urls) > 0 {
|
||
for _, url := range urls {
|
||
pls, err := playlist.MakeFromUrl(url)
|
||
if err != nil {
|
||
log.Printf("Warning: %s, skipping\n", err)
|
||
continue
|
||
}
|
||
lists = append(lists, pls)
|
||
}
|
||
}
|
||
|
||
if len(lists) == 0 || len(codes) > 0 {
|
||
ini, err := inifile.Init(app.Args.IniPath)
|
||
if err != nil {
|
||
log.Printf("Warning: %s, all --code flags will be ignored\n", err)
|
||
return lists
|
||
}
|
||
|
||
if len(codes) > 0 {
|
||
for _, plsCode := range codes {
|
||
list := ini.Lists[plsCode]
|
||
if list.Url == "" {
|
||
log.Printf("Warning: playlist [%s] not found in ini-file, skipping\n", plsCode)
|
||
continue
|
||
}
|
||
lists = append(lists, list)
|
||
}
|
||
} else {
|
||
if app.Config.Cache.IsActive {
|
||
cachedLists := c.getCachedPlaylists()
|
||
for key := range ini.Lists {
|
||
if _, ok := cachedLists[key]; ok {
|
||
continue
|
||
}
|
||
lists = append(lists, ini.Lists[key])
|
||
}
|
||
log.Printf("Found %d cached playlists\n", len(cachedLists))
|
||
} else {
|
||
lists = slices.Collect(maps.Values(ini.Lists))
|
||
}
|
||
|
||
if int(app.Args.RandomCount) > 0 && int(app.Args.RandomCount) <= len(lists) {
|
||
rand.Shuffle(len(lists), func(i int, j int) { lists[i], lists[j] = lists[j], lists[i] })
|
||
lists = lists[:app.Args.RandomCount]
|
||
}
|
||
}
|
||
}
|
||
|
||
return lists
|
||
}
|
||
|
||
// getCachedPlaylists возвращает из кеша проверенные ранее плейлисты
|
||
func (c *Checker) getCachedPlaylists() map[string]playlist.Playlist {
|
||
result := make(map[string]playlist.Playlist)
|
||
ctx := context.Background()
|
||
iter := app.Cache.Scan(ctx, 0, "*", 100).Iterator()
|
||
for iter.Next(ctx) {
|
||
key := iter.Val()
|
||
value, err := app.Cache.Get(ctx, key).Result()
|
||
if err != nil {
|
||
continue
|
||
}
|
||
var pls playlist.Playlist
|
||
if err := json.Unmarshal([]byte(value), &pls); err != nil {
|
||
continue
|
||
}
|
||
result[pls.Code] = pls
|
||
}
|
||
if err := iter.Err(); err != nil {
|
||
log.Printf("Error scanning cache: %s", err)
|
||
}
|
||
return result
|
||
}
|
||
|
||
// CheckPlaylists проверяет плейлисты и возвращает их же с результатами проверки
|
||
func (c *Checker) CheckPlaylists(ctx context.Context, lists []playlist.Playlist) (int, int) {
|
||
count := len(lists)
|
||
if count == 0 {
|
||
log.Println("There are no playlists to check")
|
||
os.Exit(0)
|
||
}
|
||
|
||
log.Printf("%d playlists will be checked\n", len(lists))
|
||
step, onlineCount, offlineCount := 0, 0, 0
|
||
|
||
for idx := range lists {
|
||
pls := lists[idx]
|
||
step++
|
||
|
||
var err error
|
||
if pls.Source == "-f" {
|
||
// direct m3u path
|
||
log.Printf("[%.3d/%.3d] Playlist from filesystem\n", step, count)
|
||
log.Printf("Reading file... (%s)\n", pls.Url)
|
||
err = pls.ReadFromFs()
|
||
} else if pls.Source == "-u" {
|
||
// direct m3u url
|
||
log.Printf("[%.3d/%.3d] Playlist [%s]\n", step, count, pls.Url)
|
||
log.Printf("Fetching... (%s)\n", pls.Url)
|
||
err = pls.Download()
|
||
} else {
|
||
// from ini
|
||
log.Printf("[%.3d/%.3d] Playlist [%s]\n", step, count, pls.Code)
|
||
log.Printf("Fetching... (%s)\n", pls.Url)
|
||
err = pls.Download()
|
||
}
|
||
|
||
if err != nil {
|
||
log.Printf("Cannot read playlist [%s]: %s\n", pls.Url, err)
|
||
offlineCount++
|
||
c.cachePlaylist(pls)
|
||
continue
|
||
}
|
||
|
||
log.Println("Parsing content...")
|
||
pls.IsOnline = true
|
||
onlineCount++
|
||
pls = pls.Parse()
|
||
|
||
log.Printf("Parsed, checking channels (%d)...\n", len(pls.Channels))
|
||
pls = c.CheckChannels(ctx, pls)
|
||
lists[idx] = pls
|
||
c.cachePlaylist(pls)
|
||
}
|
||
|
||
return onlineCount, offlineCount
|
||
}
|
||
|
||
func (c *Checker) cachePlaylist(pls playlist.Playlist) {
|
||
if !app.Config.Cache.IsActive {
|
||
return
|
||
}
|
||
|
||
jsonBytes, err := json.Marshal(pls)
|
||
if err != nil {
|
||
log.Printf("Error while saving playlist to cache: %s", err)
|
||
return
|
||
}
|
||
|
||
key := pls.Code
|
||
if key == "" {
|
||
key = "raw:" + utils.Md5str(pls.Url)
|
||
}
|
||
|
||
ctx := context.Background()
|
||
ttl := time.Duration(app.Config.Cache.Ttl) * time.Second
|
||
written := app.Cache.Set(ctx, key, string(jsonBytes), ttl)
|
||
if written.Err() != nil {
|
||
log.Printf("Error while saving playlist to cache: %s", written.Err())
|
||
return
|
||
}
|
||
|
||
log.Println("Cached sucessfully")
|
||
}
|
||
|
||
// CheckChannels проверяет каналы и возвращает их же с результатами проверки
|
||
func (c *Checker) CheckChannels(ctx context.Context, pls playlist.Playlist) playlist.Playlist {
|
||
type errorData struct {
|
||
tvChannel playlist.Channel
|
||
err error
|
||
}
|
||
|
||
count := len(pls.Channels)
|
||
if count == 0 {
|
||
log.Println("There are no channels to check, skipping")
|
||
return pls
|
||
}
|
||
|
||
pls.OnlineCount = 0
|
||
pls.OfflineCount = 0
|
||
|
||
timeout, routines := calcParameters(count)
|
||
tr := &http.Transport{
|
||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||
}
|
||
httpClient := http.Client{
|
||
Timeout: timeout,
|
||
Transport: tr,
|
||
}
|
||
chSemaphores := make(chan struct{}, routines)
|
||
chOnline := make(chan playlist.Channel, len(pls.Channels))
|
||
chOffline := make(chan playlist.Channel, len(pls.Channels))
|
||
chError := make(chan errorData, len(pls.Channels))
|
||
var wg sync.WaitGroup
|
||
|
||
startTime := time.Now()
|
||
for _, tvChannel := range pls.Channels {
|
||
wg.Add(1)
|
||
go func(tvChannel playlist.Channel) {
|
||
defer func() {
|
||
if r := recover(); r != nil {
|
||
log.Printf("Panic while checking channel '%s': %v", tvChannel.Title, r)
|
||
}
|
||
<-chSemaphores
|
||
wg.Done()
|
||
}()
|
||
chSemaphores <- struct{}{}
|
||
|
||
tvChannel.Tags = c.getTagsForChannel(tvChannel)
|
||
|
||
req, err := http.NewRequestWithContext(ctx, "GET", tvChannel.URL, nil)
|
||
if err != nil {
|
||
data := errorData{tvChannel: tvChannel, err: err}
|
||
chError <- data
|
||
return
|
||
}
|
||
|
||
req.Header.Set("User-Agent", "Mozilla/5.0 WINK/1.31.1 (AndroidTV/9) HlsWinkPlayer")
|
||
req.Header.Set("Range", "bytes=0-511") // 512 B, but sometimes servers ignore it
|
||
resp, err := httpClient.Do(req)
|
||
tvChannel.CheckedAt = time.Now().Unix()
|
||
if err != nil {
|
||
data := errorData{tvChannel: tvChannel, err: err}
|
||
chError <- data
|
||
return
|
||
}
|
||
|
||
tvChannel.Status = resp.StatusCode
|
||
tvChannel.IsOnline = tvChannel.Status < http.StatusBadRequest
|
||
tvChannel.ContentType = resp.Header.Get("Content-Type")
|
||
chunk := io.LimitReader(resp.Body, 512) // just for sure
|
||
bodyBytes, err := io.ReadAll(chunk)
|
||
if err != nil {
|
||
_ = resp.Body.Close()
|
||
data := errorData{tvChannel: tvChannel, err: err}
|
||
chError <- data
|
||
return
|
||
}
|
||
bodyString := string(bodyBytes)
|
||
_ = resp.Body.Close()
|
||
contentType := http.DetectContentType(bodyBytes)
|
||
|
||
isContentBinary := strings.Contains(contentType, "octet-stream") ||
|
||
strings.Contains(contentType, "video/")
|
||
|
||
isContentCorrect := isContentBinary ||
|
||
strings.Contains(bodyString, "#EXTM3U") ||
|
||
strings.Contains(bodyString, "#EXT-X-") ||
|
||
strings.Contains(bodyString, "<MPD ") ||
|
||
strings.Contains(bodyString, "<SegmentTemplate ") ||
|
||
strings.Contains(bodyString, "<AdaptationSet ")
|
||
|
||
if tvChannel.Status >= http.StatusBadRequest && !isContentCorrect {
|
||
tvChannel.Error = bodyString
|
||
chOffline <- tvChannel
|
||
return
|
||
}
|
||
|
||
chOnline <- tvChannel
|
||
return
|
||
}(tvChannel)
|
||
}
|
||
|
||
for idx := 1; idx <= count; idx++ {
|
||
select {
|
||
case tvChannel := <-chOnline:
|
||
tvChannel.IsOnline = true
|
||
pls.Channels[tvChannel.Id] = tvChannel
|
||
if app.Args.Verbose {
|
||
log.Printf("[%.3d/%.3d] ONLINE '%s'\n", idx, count, tvChannel.Title)
|
||
log.Printf("> Id: %s\n", tvChannel.Id)
|
||
log.Printf("> Tags: %s\n", strings.Join(tvChannel.Tags, ","))
|
||
log.Printf("> MimeType: %s\n", tvChannel.ContentType)
|
||
}
|
||
case tvChannel := <-chOffline:
|
||
pls.Channels[tvChannel.Id] = tvChannel
|
||
if app.Args.Verbose {
|
||
log.Printf("[%.3d/%.3d] OFFLINE '%s'\n", idx, count, tvChannel.Title)
|
||
log.Printf("> Id: %s\n", tvChannel.Id)
|
||
log.Printf("> Tags: %s\n", strings.Join(tvChannel.Tags, ","))
|
||
log.Printf("> Status: %d\n", tvChannel.Status)
|
||
}
|
||
case data := <-chError:
|
||
pls.Channels[data.tvChannel.Id] = data.tvChannel
|
||
if app.Args.Verbose {
|
||
log.Printf("[%.3d/%.3d] ERROR '%s'\n", idx, count, data.tvChannel.Title)
|
||
log.Printf("> Id: %s\n", data.tvChannel.Id)
|
||
log.Printf("> Tags: %s\n", strings.Join(data.tvChannel.Tags, ","))
|
||
log.Printf("> Error: %s\n", data.err)
|
||
}
|
||
}
|
||
}
|
||
|
||
wg.Wait()
|
||
close(chOnline)
|
||
close(chOffline)
|
||
close(chError)
|
||
pls.CheckedAt = time.Now().Unix()
|
||
|
||
for _, tvChannel := range pls.Channels {
|
||
if tvChannel.IsOnline {
|
||
pls.OnlineCount++
|
||
} else {
|
||
pls.OfflineCount++
|
||
}
|
||
}
|
||
|
||
log.Printf(
|
||
"Checked successfully! online=%d onlinePercent=%.2f%% offline=%d offlinePercent=%.2f%% elapsedTime=%.2fs",
|
||
pls.OnlineCount,
|
||
float32(pls.OnlineCount)/float32(len(pls.Channels))*100,
|
||
pls.OfflineCount,
|
||
float32(pls.OfflineCount)/float32(len(pls.Channels))*100,
|
||
time.Since(startTime).Seconds(),
|
||
)
|
||
|
||
return pls
|
||
}
|
||
|
||
// calcParameters вычисляет оптимальное количество горутин и таймаут запроса
|
||
func calcParameters(count int) (time.Duration, int) {
|
||
if count <= 0 {
|
||
return 10 * time.Second, 1
|
||
}
|
||
|
||
routines := count
|
||
if routines > 3000 {
|
||
routines = 3000
|
||
}
|
||
if routines < 1 {
|
||
routines = 1
|
||
}
|
||
|
||
var digits = 1
|
||
x := count
|
||
for x >= 10 {
|
||
digits++
|
||
x /= 10
|
||
}
|
||
|
||
timeout := 10 - int(math.Ceil(float64(digits)*1.5))
|
||
if timeout > 10 {
|
||
timeout = 10
|
||
}
|
||
if timeout < 1 {
|
||
timeout = 1
|
||
}
|
||
|
||
duration := time.Duration(timeout) * time.Second
|
||
log.Printf(
|
||
"Check parameters calculated count=%d timeout=%.2fs routines=%d\n",
|
||
count,
|
||
duration.Seconds(),
|
||
routines,
|
||
)
|
||
|
||
return duration, routines
|
||
}
|
||
|
||
// getTagsForChannel ищет и возвращает теги для канала
|
||
func (c *Checker) getTagsForChannel(tvChannel playlist.Channel) []string {
|
||
var foundTags []string
|
||
|
||
for _, block := range c.tagBlocks {
|
||
tags := block.GetTags(tvChannel)
|
||
if tags != nil {
|
||
foundTags = append(foundTags, tags...)
|
||
}
|
||
}
|
||
|
||
if len(foundTags) == 0 {
|
||
foundTags = append(foundTags, "untagged")
|
||
} else if len(foundTags) > 0 {
|
||
foundTags = utils.ArrayUnique(foundTags)
|
||
}
|
||
|
||
return foundTags
|
||
}
|