/* * Copyright (c) 2025, Антон Аксенов * This file is part of iptvc project * MIT License: https://git.axenov.dev/IPTV/iptvc/src/branch/master/LICENSE */ package checker import ( "axenov/iptv-checker/app" "axenov/iptv-checker/app/inifile" "axenov/iptv-checker/app/playlist" "axenov/iptv-checker/app/tagfile" "axenov/iptv-checker/app/utils" "context" "crypto/tls" "encoding/json" "io" "log" "maps" "math" "math/rand" "net/http" "os" "runtime" "slices" "strings" "sync" "time" ) var ( tagBlocks []tagfile.TagBlock ctx = context.Background() ) // PrepareListsToCheck готовит список плейлистов для проверки func PrepareListsToCheck(files []string, urls []string, codes []string) []playlist.Playlist { var lists []playlist.Playlist if len(files) > 0 { for _, filepath := range files { pls, err := playlist.MakeFromFile(filepath) if err != nil { log.Printf("Warning: %s, skipping\n", err) continue } lists = append(lists, pls) } } if len(urls) > 0 { for _, url := range urls { pls, _ := playlist.MakeFromUrl(url) lists = append(lists, pls) } } if len(lists) == 0 || len(codes) > 0 { ini, err := inifile.Init(app.Args.IniPath) if err != nil { log.Printf("Warning: %s, all --code flags will be ignored\n", err) return lists } if len(codes) > 0 { for _, plsCode := range codes { list := ini.Lists[plsCode] if list.Url == "" { log.Printf("Warning: playlist [%s] not found in ini-file, skipping\n", plsCode) continue } lists = append(lists, list) } } else { if app.Config.Cache.IsActive { cachedLists := getCachedPlaylists() for key := range ini.Lists { if _, ok := cachedLists[key]; ok { continue } lists = append(lists, ini.Lists[key]) } log.Printf("Found %d cached playlists\n", len(cachedLists)) } else { lists = slices.Collect(maps.Values(ini.Lists)) } if int(app.Args.RandomCount) > 0 && int(app.Args.RandomCount) <= len(lists) { rand.Shuffle(len(lists), func(i int, j int) { lists[i], lists[j] = lists[j], lists[i] }) lists = lists[:app.Args.RandomCount] } } } return lists } // getCachedPlaylists возвращает из кеша проверенные ранее плейлисты func getCachedPlaylists() map[string]playlist.Playlist { result := make(map[string]playlist.Playlist) keys := app.Cache.Keys(ctx, "*") for _, key := range keys.Val() { value := app.Cache.Get(ctx, key).Val() var pls playlist.Playlist _ = json.Unmarshal([]byte(value), &pls) result[pls.Code] = pls } return result } // CheckPlaylists проверяет плейлисты и возвращает их же с результатами проверки func CheckPlaylists(lists []playlist.Playlist) (int, int) { count := len(lists) if count == 0 { log.Println("There are no playlists to check") os.Exit(0) } log.Printf("%d playlists will be checked\n", len(lists)) step, onlineCount, offlineCount := 0, 0, 0 tagBlocks = tagfile.Init(app.Args.TagsPath) for idx := range lists { pls := lists[idx] step++ var err error if pls.Source == "-f" { // direct m3u path log.Printf("[%.3d/%.3d] Playlist from filesystem\n", step, count) log.Printf("Reading file... (%s)\n", pls.Url) err = pls.ReadFromFs() } else if pls.Source == "-u" { // direct m3u url log.Printf("[%.3d/%.3d] Playlist [%s]\n", step, count, pls.Url) log.Printf("Fetching... (%s)\n", pls.Url) err = pls.Download() } else { // from ini log.Printf("[%.3d/%.3d] Playlist [%s]\n", step, count, pls.Code) log.Printf("Fetching... (%s)\n", pls.Url) err = pls.Download() } if err != nil { log.Printf("Cannot read playlist [%s]: %s\n", pls.Url, err) offlineCount++ cachePlaylist(pls) continue } log.Println("Parsing content...") pls.IsOnline = true onlineCount++ pls = pls.Parse() log.Printf("Parsed, checking channels (%d)...\n", len(pls.Channels)) pls = CheckChannels(pls) lists[idx] = pls cachePlaylist(pls) } return onlineCount, offlineCount } func cachePlaylist(pls playlist.Playlist) { if app.Config.Cache.IsActive { jsonBytes, err := json.Marshal(pls) if err != nil { log.Printf("Error while saving playlist to cache: %s", err) } ttl := time.Duration(app.Config.Cache.Ttl) * time.Second written := app.Cache.Set(ctx, pls.Code, string(jsonBytes), ttl) if written.Err() != nil { log.Printf("Error while saving playlist to cache: %s", err) } log.Println("Cached sucessfully") } } // CheckChannels проверяет каналы и возвращает их же с результатами проверки func CheckChannels(pls playlist.Playlist) playlist.Playlist { type errorData struct { tvChannel playlist.Channel err error } count := len(pls.Channels) if count == 0 { log.Println("There are no channels to check, skipping") return pls } timeout, routines := calcParameters(count) httpClient := http.Client{Timeout: timeout} chSemaphores := make(chan struct{}, routines) chOnline := make(chan playlist.Channel, len(pls.Channels)) chOffline := make(chan playlist.Channel, len(pls.Channels)) chError := make(chan errorData, len(pls.Channels)) var wg sync.WaitGroup startTime := time.Now() for _, tvChannel := range pls.Channels { wg.Add(1) go func(tvChannel playlist.Channel) { chSemaphores <- struct{}{} defer func() { <-chSemaphores wg.Done() }() tvChannel.Tags = getTagsForChannel(tvChannel) http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} req, err := http.NewRequest("GET", tvChannel.URL, nil) if err != nil { data := errorData{tvChannel: tvChannel, err: err} chError <- data return } //TODO user-agent req.Header.Set("User-Agent", "Mozilla/5.0 WINK/1.31.1 (AndroidTV/9) HlsWinkPlayer") req.Header.Add("Range", "bytes=0-1023") // 1 Kb, but sometimes servers ignore it resp, err := httpClient.Do(req) tvChannel.CheckedAt = time.Now().Unix() if err != nil { data := errorData{tvChannel: tvChannel, err: err} chError <- data return } tvChannel.Status = resp.StatusCode tvChannel.IsOnline = tvChannel.Status < http.StatusBadRequest tvChannel.ContentType = resp.Header.Get("Content-Type") chunk := io.LimitReader(resp.Body, 1024) // just for sure bodyBytes, _ := io.ReadAll(chunk) bodyString := string(bodyBytes) _ = resp.Body.Close() contentType := http.DetectContentType(bodyBytes) isContentBinary := strings.Contains(contentType, "octet-stream") || strings.Contains(contentType, "video/") isContentCorrect := isContentBinary || strings.Contains(bodyString, "#EXTM3U") || strings.Contains(bodyString, "#EXT-X-") || strings.Contains(bodyString, "= http.StatusBadRequest && !isContentCorrect { tvChannel.Error = bodyString chOffline <- tvChannel return } chOnline <- tvChannel return }(tvChannel) } for idx := 1; idx <= count; idx++ { select { case tvChannel := <-chOnline: tvChannel.IsOnline = true pls.Channels[tvChannel.Id] = tvChannel if app.Args.Verbose { log.Printf("[%.3d/%.3d] ONLINE '%s'\n", idx, count, tvChannel.Title) log.Printf("> Id: %s\n", tvChannel.Id) log.Printf("> Tags: %s\n", strings.Join(tvChannel.Tags, ",")) log.Printf("> MimeType: %s\n", tvChannel.ContentType) } case tvChannel := <-chOffline: pls.Channels[tvChannel.Id] = tvChannel if app.Args.Verbose { log.Printf("[%.3d/%.3d] OFFLINE '%s'\n", idx, count, tvChannel.Title) log.Printf("> Id: %s\n", tvChannel.Id) log.Printf("> Tags: %s\n", strings.Join(tvChannel.Tags, ",")) log.Printf("> Status: %d\n", tvChannel.Status) } case data := <-chError: pls.Channels[data.tvChannel.Id] = data.tvChannel if app.Args.Verbose { log.Printf("[%.3d/%.3d] ERROR '%s'\n", idx, count, data.tvChannel.Title) log.Printf("> Id: %s\n", data.tvChannel.Id) log.Printf("> Tags: %s\n", strings.Join(data.tvChannel.Tags, ",")) log.Printf("> Error: %s\n", data.err) } } } wg.Wait() close(chOnline) close(chOffline) close(chError) pls.CheckedAt = time.Now().Unix() for _, tvChannel := range pls.Channels { if tvChannel.IsOnline { pls.OnlineCount++ } else { pls.OfflineCount++ } } log.Printf( "Checked successfully! online=%d onlinePercent=%.2f%% offline=%d offlinePercent=%.2f%% elapsedTime=%.2fs", pls.OnlineCount, float32(pls.OnlineCount)/float32(len(pls.Channels))*100, pls.OfflineCount, float32(pls.OfflineCount)/float32(len(pls.Channels))*100, time.Since(startTime).Seconds(), ) return pls } // calcParameters вычисляет оптимальное количество горутин и таймаут запроса func calcParameters(count int) (time.Duration, int) { routines := count if routines > 3000 { routines = 3000 } if routines < 1 { routines = 1 } var digits = 1 x := count for x >= 10 { digits++ x /= 10 } timeout := 10 - int(math.Ceil(float64(digits)*1.5)) if timeout > 10 { timeout = 10 } if timeout < 1 { timeout = 1 } duration := time.Duration(timeout) * time.Second log.Printf( "Check parameters calculated count=%d timeout=%.2fs routines=%d\n", count, duration.Seconds(), routines, ) return duration, routines } // getTagsForChannel ищет и возвращает теги для канала func getTagsForChannel(tvChannel playlist.Channel) []string { var foundTags []string for _, block := range tagBlocks { tags := block.GetTags(tvChannel) if tags != nil { foundTags = append(foundTags, tags...) } } if len(foundTags) == 0 { foundTags = append(foundTags, "untagged") } else if len(foundTags) > 0 { foundTags = utils.ArrayUnique(foundTags) } return foundTags }