409 lines
11 KiB
Go
409 lines
11 KiB
Go
/*
|
|
* Copyright (c) 2025-2026, Антон Аксенов
|
|
* This file is part of iptvc project
|
|
* MIT License: https://git.axenov.dev/IPTV/iptvc/src/branch/master/LICENSE
|
|
*/
|
|
|
|
package checker
|
|
|
|
import (
|
|
"axenov/iptv-checker/app"
|
|
"axenov/iptv-checker/app/inifile"
|
|
"axenov/iptv-checker/app/playlist"
|
|
"axenov/iptv-checker/app/tagfile"
|
|
"axenov/iptv-checker/app/utils"
|
|
"context"
|
|
"crypto/tls"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"maps"
|
|
"math/rand"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
tagBlocks []tagfile.TagBlock
|
|
ctx = context.Background()
|
|
OnPlaylistChecked func(playlist.Playlist) // вызывается после проверки каждого плейлиста
|
|
)
|
|
|
|
// PrepareListsToCheck готовит список плейлистов для проверки
|
|
func PrepareListsToCheck(files []string, urls []string, codes []string) []playlist.Playlist {
|
|
var lists []playlist.Playlist
|
|
|
|
if len(files) > 0 {
|
|
for _, filepath := range files {
|
|
pls, err := playlist.MakeFromFile(filepath)
|
|
if err != nil {
|
|
log.Printf("Warning: %s, skipping\n", err)
|
|
continue
|
|
}
|
|
|
|
lists = append(lists, pls)
|
|
}
|
|
}
|
|
|
|
if len(urls) > 0 {
|
|
for _, url := range urls {
|
|
pls, _ := playlist.MakeFromUrl(url)
|
|
lists = append(lists, pls)
|
|
}
|
|
}
|
|
|
|
if len(lists) == 0 || len(codes) > 0 {
|
|
ini, err := inifile.Init(app.Config.App.Playlists)
|
|
if err != nil {
|
|
log.Printf("Warning: %s, all --code flags will be ignored\n", err)
|
|
return lists
|
|
}
|
|
|
|
if len(codes) > 0 {
|
|
for _, plsCode := range codes {
|
|
list := ini.Lists[plsCode]
|
|
if list.Url == "" {
|
|
log.Printf("Warning: playlist [%s] not found in ini-file, skipping\n", plsCode)
|
|
continue
|
|
}
|
|
lists = append(lists, list)
|
|
}
|
|
} else {
|
|
if app.Config.Cache.IsActive {
|
|
cachedLists := getCachedPlaylists()
|
|
for key := range ini.Lists {
|
|
if _, ok := cachedLists[key]; ok {
|
|
continue
|
|
}
|
|
lists = append(lists, ini.Lists[key])
|
|
}
|
|
log.Printf("Found %d cached playlists\n", len(cachedLists))
|
|
} else {
|
|
lists = slices.Collect(maps.Values(ini.Lists))
|
|
}
|
|
|
|
if int(app.Args.RandomCount) > 0 && int(app.Args.RandomCount) <= len(lists) {
|
|
rand.Shuffle(len(lists), func(i int, j int) { lists[i], lists[j] = lists[j], lists[i] })
|
|
lists = lists[:app.Args.RandomCount]
|
|
}
|
|
}
|
|
}
|
|
|
|
return lists
|
|
}
|
|
|
|
// getCachedPlaylists возвращает из кеша проверенные ранее плейлисты
|
|
func getCachedPlaylists() map[string]playlist.Playlist {
|
|
result := make(map[string]playlist.Playlist)
|
|
keys := app.Cache.Keys(ctx, "*")
|
|
for _, key := range keys.Val() {
|
|
value := app.Cache.Get(ctx, key).Val()
|
|
var pls playlist.Playlist
|
|
_ = json.Unmarshal([]byte(value), &pls)
|
|
result[pls.Code] = pls
|
|
}
|
|
return result
|
|
}
|
|
|
|
// CheckPlaylists проверяет плейлисты и возвращает их же с результатами проверки.
|
|
// Параметры (user-agent, timeout, cooldown, per-routine) берутся из config.Check.Playlists.
|
|
func CheckPlaylists(lists []playlist.Playlist) (int, int) {
|
|
count := len(lists)
|
|
if count == 0 {
|
|
log.Println("There are no playlists to check")
|
|
return 0, 0
|
|
}
|
|
|
|
pc := app.Config.Check.Playlists
|
|
maxRoutines := pc.MaxRoutines
|
|
if maxRoutines < 1 {
|
|
maxRoutines = 1
|
|
}
|
|
|
|
log.Printf("%d playlists will be checked (max-routines=%d)\n", count, maxRoutines)
|
|
tagBlocks = tagfile.Init(app.Config.App.Tags)
|
|
|
|
type checkResult struct {
|
|
idx int
|
|
pls playlist.Playlist
|
|
online bool
|
|
}
|
|
|
|
sem := make(chan struct{}, maxRoutines)
|
|
var wg sync.WaitGroup
|
|
results := make([]checkResult, count)
|
|
|
|
for idx := range lists {
|
|
wg.Add(1)
|
|
go func(idx int) {
|
|
sem <- struct{}{}
|
|
defer func() {
|
|
<-sem
|
|
wg.Done()
|
|
}()
|
|
|
|
pls := lists[idx]
|
|
pls.CheckedAt = time.Now().Unix()
|
|
|
|
userAgent := pc.UserAgent.Pick()
|
|
timeout := time.Duration(pc.Timeout) * time.Millisecond
|
|
|
|
var err error
|
|
if pls.Source == "-f" {
|
|
log.Printf("Playlist from filesystem (%s)\n", pls.Url)
|
|
err = pls.ReadFromFs()
|
|
} else {
|
|
log.Printf("Fetching playlist [%s] (%s)\n", pls.Code, pls.Url)
|
|
err = pls.Download(userAgent, timeout)
|
|
}
|
|
|
|
if err != nil {
|
|
log.Printf("Cannot read playlist [%s]: %s\n", pls.Url, err)
|
|
pls.IsOnline = false
|
|
results[idx] = checkResult{idx: idx, pls: pls, online: false}
|
|
cachePlaylist(pls)
|
|
if OnPlaylistChecked != nil {
|
|
OnPlaylistChecked(pls)
|
|
}
|
|
return
|
|
}
|
|
|
|
log.Println("Parsing content...")
|
|
pls.IsOnline = true
|
|
pls = pls.Parse()
|
|
|
|
log.Printf("Parsed, checking channels (%d)...\n", len(pls.Channels))
|
|
pls = CheckChannels(pls)
|
|
results[idx] = checkResult{idx: idx, pls: pls, online: true}
|
|
cachePlaylist(pls)
|
|
if OnPlaylistChecked != nil {
|
|
OnPlaylistChecked(pls)
|
|
}
|
|
|
|
// one-cooldown: задержка после проверки каждого плейлиста
|
|
// pc.OneCooldown хранится в миллисекундах, поэтому переводим в time.Millisecond
|
|
oneCd := time.Duration(pc.OneCooldown.Value()) * time.Millisecond
|
|
if oneCd > 0 {
|
|
time.Sleep(oneCd)
|
|
}
|
|
}(idx)
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
onlineCount, offlineCount := 0, 0
|
|
for _, r := range results {
|
|
lists[r.idx] = r.pls
|
|
if r.online {
|
|
onlineCount++
|
|
} else {
|
|
offlineCount++
|
|
}
|
|
}
|
|
|
|
return onlineCount, offlineCount
|
|
}
|
|
|
|
func cachePlaylist(pls playlist.Playlist) {
|
|
if !app.Config.Cache.IsActive {
|
|
return
|
|
}
|
|
|
|
jsonBytes, err := json.Marshal(pls)
|
|
if err != nil {
|
|
log.Printf("Error while saving playlist to cache: %s", err)
|
|
}
|
|
|
|
ttl := time.Duration(app.Config.Cache.Ttl) * time.Second
|
|
written := app.Cache.Set(ctx, pls.Code, string(jsonBytes), ttl)
|
|
if written.Err() != nil {
|
|
log.Printf("Error while saving playlist to cache: %s", err)
|
|
}
|
|
|
|
log.Println("Cached sucessfully")
|
|
}
|
|
|
|
// CheckChannels проверяет каналы и возвращает их же с результатами проверки.
|
|
// Параметры (user-agent, timeout, byte-range, cooldown, per-routine) берутся из config.Check.Channels.
|
|
func CheckChannels(pls playlist.Playlist) playlist.Playlist {
|
|
type errorData struct {
|
|
tvChannel playlist.Channel
|
|
err error
|
|
}
|
|
|
|
count := len(pls.Channels)
|
|
if count == 0 {
|
|
log.Println("There are no channels to check, skipping")
|
|
return pls
|
|
}
|
|
|
|
pls.OnlineCount = 0
|
|
pls.OfflineCount = 0
|
|
|
|
cc := app.Config.Check.Channels
|
|
timeoutMs := cc.Timeout
|
|
byteRange := cc.ByteRange
|
|
if byteRange <= 0 {
|
|
byteRange = 512
|
|
}
|
|
maxRoutines := cc.MaxRoutines
|
|
if maxRoutines < 1 {
|
|
maxRoutines = 1
|
|
}
|
|
|
|
timeout := time.Duration(timeoutMs) * time.Millisecond
|
|
httpClient := http.Client{Timeout: timeout}
|
|
chSemaphores := make(chan struct{}, maxRoutines)
|
|
chOnline := make(chan playlist.Channel, len(pls.Channels))
|
|
chOffline := make(chan playlist.Channel, len(pls.Channels))
|
|
chError := make(chan errorData, len(pls.Channels))
|
|
var wg sync.WaitGroup
|
|
|
|
log.Printf("Check channels parameters: timeout=%.2fs byte-range=%d max-routines=%d\n", float64(timeoutMs)/1000, byteRange, maxRoutines)
|
|
|
|
startTime := time.Now()
|
|
for _, tvChannel := range pls.Channels {
|
|
wg.Add(1)
|
|
go func(tvChannel playlist.Channel) {
|
|
chSemaphores <- struct{}{}
|
|
defer func() {
|
|
<-chSemaphores
|
|
wg.Done()
|
|
}()
|
|
|
|
tvChannel.Tags = getTagsForChannel(tvChannel)
|
|
|
|
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
|
req, err := http.NewRequest("GET", tvChannel.URL, nil)
|
|
if err != nil {
|
|
data := errorData{tvChannel: tvChannel, err: err}
|
|
chError <- data
|
|
return
|
|
}
|
|
|
|
req.Header.Set("User-Agent", cc.UserAgent.Pick())
|
|
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", byteRange-1))
|
|
resp, err := httpClient.Do(req)
|
|
tvChannel.CheckedAt = time.Now().Unix()
|
|
if err != nil {
|
|
data := errorData{tvChannel: tvChannel, err: err}
|
|
chError <- data
|
|
return
|
|
}
|
|
|
|
tvChannel.Status = resp.StatusCode
|
|
tvChannel.IsOnline = tvChannel.Status < http.StatusBadRequest
|
|
tvChannel.ContentType = resp.Header.Get("Content-Type")
|
|
chunk := io.LimitReader(resp.Body, int64(byteRange))
|
|
bodyBytes, _ := io.ReadAll(chunk)
|
|
bodyString := string(bodyBytes)
|
|
_ = resp.Body.Close()
|
|
contentType := http.DetectContentType(bodyBytes)
|
|
|
|
isContentBinary := strings.Contains(contentType, "octet-stream") ||
|
|
strings.Contains(contentType, "video/")
|
|
|
|
isContentCorrect := isContentBinary ||
|
|
strings.Contains(bodyString, "#EXTM3U") ||
|
|
strings.Contains(bodyString, "#EXT-X-") ||
|
|
strings.Contains(bodyString, "<MPD ") ||
|
|
strings.Contains(bodyString, "<SegmentTemplate ") ||
|
|
strings.Contains(bodyString, "<AdaptationSet ")
|
|
|
|
if tvChannel.Status >= http.StatusBadRequest && !isContentCorrect {
|
|
tvChannel.Error = bodyString
|
|
chOffline <- tvChannel
|
|
return
|
|
}
|
|
|
|
chOnline <- tvChannel
|
|
|
|
// cooldown: задержка после проверки каждого канала
|
|
// cc.Cooldown хранится в миллисекундах, поэтому переводим в time.Millisecond
|
|
cd := time.Duration(cc.Cooldown.Value()) * time.Millisecond
|
|
if cd > 0 {
|
|
time.Sleep(cd)
|
|
}
|
|
}(tvChannel)
|
|
}
|
|
|
|
for idx := 1; idx <= count; idx++ {
|
|
select {
|
|
case tvChannel := <-chOnline:
|
|
tvChannel.IsOnline = true
|
|
pls.Channels[tvChannel.Id] = tvChannel
|
|
if app.Args.Verbose {
|
|
log.Printf("[%.3d/%.3d] ONLINE '%s'\n", idx, count, tvChannel.Title)
|
|
log.Printf("> Id: %s\n", tvChannel.Id)
|
|
log.Printf("> Tags: %s\n", strings.Join(tvChannel.Tags, ","))
|
|
log.Printf("> MimeType: %s\n", tvChannel.ContentType)
|
|
}
|
|
case tvChannel := <-chOffline:
|
|
pls.Channels[tvChannel.Id] = tvChannel
|
|
if app.Args.Verbose {
|
|
log.Printf("[%.3d/%.3d] OFFLINE '%s'\n", idx, count, tvChannel.Title)
|
|
log.Printf("> Id: %s\n", tvChannel.Id)
|
|
log.Printf("> Tags: %s\n", strings.Join(tvChannel.Tags, ","))
|
|
log.Printf("> Status: %d\n", tvChannel.Status)
|
|
}
|
|
case data := <-chError:
|
|
pls.Channels[data.tvChannel.Id] = data.tvChannel
|
|
if app.Args.Verbose {
|
|
log.Printf("[%.3d/%.3d] ERROR '%s'\n", idx, count, data.tvChannel.Title)
|
|
log.Printf("> Id: %s\n", data.tvChannel.Id)
|
|
log.Printf("> Tags: %s\n", strings.Join(data.tvChannel.Tags, ","))
|
|
log.Printf("> Error: %s\n", data.err)
|
|
}
|
|
}
|
|
}
|
|
|
|
wg.Wait()
|
|
close(chOnline)
|
|
close(chOffline)
|
|
close(chError)
|
|
pls.CheckedAt = time.Now().Unix()
|
|
|
|
for _, tvChannel := range pls.Channels {
|
|
if tvChannel.IsOnline {
|
|
pls.OnlineCount++
|
|
} else {
|
|
pls.OfflineCount++
|
|
}
|
|
}
|
|
|
|
log.Printf(
|
|
"Checked successfully! online=%d onlinePercent=%.2f%% offline=%d offlinePercent=%.2f%% elapsedTime=%.2fs",
|
|
pls.OnlineCount,
|
|
float32(pls.OnlineCount)/float32(len(pls.Channels))*100,
|
|
pls.OfflineCount,
|
|
float32(pls.OfflineCount)/float32(len(pls.Channels))*100,
|
|
time.Since(startTime).Seconds(),
|
|
)
|
|
|
|
return pls
|
|
}
|
|
|
|
// getTagsForChannel ищет и возвращает теги для канала
|
|
func getTagsForChannel(tvChannel playlist.Channel) []string {
|
|
var foundTags []string
|
|
|
|
for _, block := range tagBlocks {
|
|
tags := block.GetTags(tvChannel)
|
|
if tags != nil {
|
|
foundTags = append(foundTags, tags...)
|
|
}
|
|
}
|
|
|
|
if len(foundTags) == 0 {
|
|
foundTags = append(foundTags, "untagged")
|
|
} else if len(foundTags) > 0 {
|
|
foundTags = utils.ArrayUnique(foundTags)
|
|
}
|
|
|
|
return foundTags
|
|
}
|