This commit is contained in:
336
app/checker/checker.go
Normal file
336
app/checker/checker.go
Normal file
@@ -0,0 +1,336 @@
|
||||
/*
|
||||
* Copyright (c) 2025, Антон Аксенов
|
||||
* This file is part of iptvc project
|
||||
* MIT License: https://git.axenov.dev/IPTV/iptvc/src/branch/master/LICENSE
|
||||
*/
|
||||
|
||||
package checker
|
||||
|
||||
import (
|
||||
"axenov/iptv-checker/app"
|
||||
"axenov/iptv-checker/app/inifile"
|
||||
"axenov/iptv-checker/app/playlist"
|
||||
"axenov/iptv-checker/app/tagfile"
|
||||
"axenov/iptv-checker/app/utils"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"maps"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var tagBlocks []tagfile.TagBlock
|
||||
|
||||
// PrepareListsToCheck готовит список плейлистов для проверки
|
||||
func PrepareListsToCheck(files []string, urls []string, codes []string) []playlist.Playlist {
|
||||
var lists []playlist.Playlist
|
||||
|
||||
if len(files) > 0 {
|
||||
for _, filepath := range files {
|
||||
pls, err := playlist.MakeFromFile(filepath)
|
||||
if err != nil {
|
||||
log.Printf("Warning: %s, skipping\n", err)
|
||||
continue
|
||||
}
|
||||
|
||||
lists = append(lists, pls)
|
||||
}
|
||||
}
|
||||
|
||||
if len(urls) > 0 {
|
||||
for _, url := range urls {
|
||||
pls, _ := playlist.MakeFromUrl(url)
|
||||
lists = append(lists, pls)
|
||||
}
|
||||
}
|
||||
|
||||
ini, err := inifile.Init(app.Args.IniPath)
|
||||
if err != nil {
|
||||
log.Printf("Warning: %s, all --code flags will be ignored\n", err)
|
||||
return lists
|
||||
}
|
||||
|
||||
if len(codes) > 0 {
|
||||
for _, plsCode := range codes {
|
||||
list := ini.Lists[plsCode]
|
||||
if list.Url == "" {
|
||||
log.Printf("Warning: playlist [%s] not found in ini-file, skipping\n", plsCode)
|
||||
continue
|
||||
}
|
||||
lists = append(lists, list)
|
||||
}
|
||||
} else {
|
||||
lists = slices.Collect(maps.Values(ini.Lists))
|
||||
if int(app.Args.RandomCount) > 0 && int(app.Args.RandomCount) <= len(lists) {
|
||||
rand.Shuffle(len(lists), func(i int, j int) { lists[i], lists[j] = lists[j], lists[i] })
|
||||
lists = lists[:app.Args.RandomCount]
|
||||
}
|
||||
}
|
||||
|
||||
return lists
|
||||
}
|
||||
|
||||
// CheckPlaylists проверяет плейлисты и возвращает их же с результатами проверки
|
||||
func CheckPlaylists(lists []playlist.Playlist) {
|
||||
step := 0
|
||||
count := len(lists)
|
||||
tagBlocks = tagfile.Init(app.Args.TagsPath)
|
||||
|
||||
if count == 0 {
|
||||
log.Println("There are no playlists to check")
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
for idx := range lists {
|
||||
pls := lists[idx]
|
||||
step++
|
||||
|
||||
var err error
|
||||
if pls.Source == "-f" {
|
||||
// direct m3u path
|
||||
log.Printf("[%.3d/%.3d] Playlist from filesystem\n", step, count)
|
||||
log.Printf("Reading file... (%s)\n", pls.Url)
|
||||
err = pls.ReadFromFs()
|
||||
} else if pls.Source == "-u" {
|
||||
// direct m3u url
|
||||
log.Printf("[%.3d/%.3d] Playlist [%s]\n", step, count, pls.Url)
|
||||
log.Printf("Fetching... (%s)\n", pls.Url)
|
||||
err = pls.Download()
|
||||
} else {
|
||||
// from ini
|
||||
log.Printf("[%.3d/%.3d] Playlist [%s]\n", step, count, pls.Code)
|
||||
log.Printf("Fetching... (%s)\n", pls.Url)
|
||||
err = pls.Download()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Cannot read playlist [%s]: %s", pls.Url, err)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Println("Parsing content...")
|
||||
pls.IsOnline = true
|
||||
pls = pls.Parse()
|
||||
|
||||
log.Printf("Parsed, checking channels (%d)...\n", len(pls.Channels))
|
||||
pls = CheckChannels(pls)
|
||||
|
||||
lists[idx] = pls
|
||||
}
|
||||
|
||||
if app.Args.NeedJson {
|
||||
marshal, _ := json.Marshal(lists)
|
||||
fmt.Println(string(marshal))
|
||||
}
|
||||
}
|
||||
|
||||
// CheckChannels проверяет каналы и возвращает их же с результатами проверки
|
||||
func CheckChannels(pls playlist.Playlist) playlist.Playlist {
|
||||
type errorData struct {
|
||||
tvChannel playlist.Channel
|
||||
err error
|
||||
}
|
||||
|
||||
count := len(pls.Channels)
|
||||
timeout, routines := calcParameters(count)
|
||||
httpClient := http.Client{Timeout: timeout}
|
||||
chSemaphores := make(chan struct{}, routines)
|
||||
chOnline := make(chan playlist.Channel, len(pls.Channels))
|
||||
chOffline := make(chan playlist.Channel, len(pls.Channels))
|
||||
chError := make(chan errorData, len(pls.Channels))
|
||||
var wg sync.WaitGroup
|
||||
|
||||
startTime := time.Now()
|
||||
for _, tvChannel := range pls.Channels {
|
||||
wg.Add(1)
|
||||
go func(tvChannel playlist.Channel) {
|
||||
chSemaphores <- struct{}{}
|
||||
defer func() {
|
||||
<-chSemaphores
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
tvChannel.Tags = getTagsForChannel(tvChannel)
|
||||
|
||||
req, err := http.NewRequest("GET", tvChannel.URL, nil)
|
||||
tvChannel.CheckedAt = time.Now().Unix()
|
||||
if err != nil {
|
||||
data := errorData{tvChannel: tvChannel, err: err}
|
||||
chError <- data
|
||||
return
|
||||
}
|
||||
|
||||
//TODO user-agent
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 WINK/1.31.1 (AndroidTV/9) HlsWinkPlayer")
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
data := errorData{tvChannel: tvChannel, err: err}
|
||||
chError <- data
|
||||
return
|
||||
}
|
||||
|
||||
tvChannel.Status = resp.StatusCode
|
||||
tvChannel.IsOnline = tvChannel.Status < http.StatusBadRequest
|
||||
tvChannel.ContentType = resp.Header.Get("Content-Type")
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
bodyString := string(bodyBytes)
|
||||
resp.Body.Close()
|
||||
contentType := http.DetectContentType(bodyBytes)
|
||||
|
||||
isContentBinary := strings.Contains(contentType, "octet-stream") ||
|
||||
strings.Contains(contentType, "video/")
|
||||
|
||||
isContentCorrect := isContentBinary ||
|
||||
strings.Contains(bodyString, "#EXTM3U") ||
|
||||
strings.Contains(bodyString, "<SegmentTemplate")
|
||||
|
||||
if tvChannel.Status >= http.StatusBadRequest || !isContentCorrect {
|
||||
tvChannel.Error = bodyString
|
||||
chOffline <- tvChannel
|
||||
return
|
||||
}
|
||||
|
||||
if isContentBinary {
|
||||
tvChannel.Content = "binary"
|
||||
} else {
|
||||
tvChannel.Content = bodyString
|
||||
}
|
||||
|
||||
chOnline <- tvChannel
|
||||
return
|
||||
}(tvChannel)
|
||||
}
|
||||
|
||||
for idx := 1; idx <= count; idx++ {
|
||||
select {
|
||||
case tvChannel := <-chOnline:
|
||||
tvChannel.IsOnline = true
|
||||
pls.OnlineCount++
|
||||
pls.Channels[tvChannel.Id] = tvChannel
|
||||
if app.Args.Verbose {
|
||||
log.Printf("[%.3d/%.3d] ONLINE '%s'\n", idx, count, tvChannel.Title)
|
||||
log.Printf("> Id: %s\n", tvChannel.Id)
|
||||
log.Printf("> Tags: %s\n", strings.Join(tvChannel.Tags, ","))
|
||||
log.Printf("> MimeType: %s\n", tvChannel.ContentType)
|
||||
}
|
||||
case tvChannel := <-chOffline:
|
||||
pls.OfflineCount++
|
||||
pls.Channels[tvChannel.Id] = tvChannel
|
||||
if app.Args.Verbose {
|
||||
log.Printf("[%.3d/%.3d] OFFLINE '%s'\n", idx, count, tvChannel.Title)
|
||||
log.Printf("> Id: %s\n", tvChannel.Id)
|
||||
log.Printf("> Tags: %s\n", strings.Join(tvChannel.Tags, ","))
|
||||
log.Printf("> Status: %d\n", tvChannel.Status)
|
||||
}
|
||||
case data := <-chError:
|
||||
pls.OfflineCount++
|
||||
pls.Channels[data.tvChannel.Id] = data.tvChannel
|
||||
if app.Args.Verbose {
|
||||
log.Printf("[%.3d/%.3d] ERROR '%s'\n", idx, count, data.tvChannel.Title)
|
||||
log.Printf("> Id: %s\n", data.tvChannel.Id)
|
||||
log.Printf("> Tags: %s\n", strings.Join(data.tvChannel.Tags, ","))
|
||||
log.Printf("> Error: %s\n", data.err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(chOnline)
|
||||
close(chOffline)
|
||||
close(chError)
|
||||
pls.CheckedAt = time.Now().Unix()
|
||||
|
||||
log.Printf(
|
||||
"Checked successfully! online=%d onlinePercent=%.2f%% offline=%d offlinePercent=%.2f%% elapsedTime=%.2fs",
|
||||
pls.OnlineCount,
|
||||
float64(pls.OnlineCount)/float64(len(pls.Channels))*100,
|
||||
pls.OfflineCount,
|
||||
float64(pls.OfflineCount)/float64(len(pls.Channels))*100,
|
||||
time.Since(startTime).Seconds(),
|
||||
)
|
||||
|
||||
return pls
|
||||
}
|
||||
|
||||
// calcParameters вычисляет оптимальное количество горутин и таймаут запроса
|
||||
func calcParameters(count int) (time.Duration, int) {
|
||||
// коэффициент нагрузки
|
||||
var k float32
|
||||
// чем ниже, тем больше горутин, меньше таймаут, быстрее проверка, хуже результаты
|
||||
// чем выше, тем меньше горутин, больше таймаут, медленнее проверка, лучше результаты
|
||||
|
||||
switch true {
|
||||
case count >= 4000:
|
||||
k = 5
|
||||
case count >= 3000:
|
||||
k = 4.5
|
||||
case count >= 2500:
|
||||
k = 4
|
||||
case count >= 2000:
|
||||
k = 3.5
|
||||
case count >= 1500:
|
||||
k = 3
|
||||
case count >= 1000:
|
||||
k = 2.5
|
||||
case count >= 500:
|
||||
k = 2
|
||||
case count >= 100:
|
||||
k = 1.5
|
||||
default:
|
||||
k = 1
|
||||
}
|
||||
|
||||
routines := int(float32(count) / k / float32(runtime.NumCPU()))
|
||||
if routines > 500 {
|
||||
routines = 500
|
||||
}
|
||||
if routines < 1 {
|
||||
routines = 1
|
||||
}
|
||||
|
||||
timeout := 10/k + 2
|
||||
if timeout > 10 {
|
||||
timeout = 10
|
||||
}
|
||||
if timeout < 1 {
|
||||
timeout = 1
|
||||
}
|
||||
|
||||
duration := time.Duration(timeout) * time.Second
|
||||
log.Printf(
|
||||
"Check parameters calculated: count=%d timeout=%.2fs routines=%d\n",
|
||||
count,
|
||||
duration.Seconds(),
|
||||
routines,
|
||||
)
|
||||
|
||||
return duration, routines
|
||||
}
|
||||
|
||||
// getTagsForChannel ищет и возвращает теги для канала
|
||||
func getTagsForChannel(tvChannel playlist.Channel) []string {
|
||||
var foundTags []string
|
||||
|
||||
for _, block := range tagBlocks {
|
||||
tags := block.GetTags(tvChannel)
|
||||
if tags != nil {
|
||||
foundTags = append(foundTags, tags...)
|
||||
}
|
||||
}
|
||||
|
||||
if len(foundTags) == 0 {
|
||||
foundTags = append(foundTags, "untagged")
|
||||
} else if len(foundTags) > 0 {
|
||||
foundTags = utils.ArrayUnique(foundTags)
|
||||
}
|
||||
|
||||
return foundTags
|
||||
}
|
||||
Reference in New Issue
Block a user