upload

2021-09-01 02:57:54 +05:00
parent 9df940f1fd
commit bf3c3712dd
222 changed files with 1007430 additions and 0 deletions
--- a/helpers/aria2.py
+++ b/helpers/aria2.py
@@ -0,0 +1,369 @@
+import os
+import shutil
+import subprocess
+import sys
+import re
+import logging
+from configs.config import tool
+from helpers.ripprocess import ripprocess
+
+
+class aria2Error(Exception):
+    pass
+
+
+class aria2_moded:
+    def __init__(self, aria2_download_command):
+        self.logger = logging.getLogger(__name__)
+        self.aria2_download_command = aria2_download_command
+        self.env = self.aria2DisableProxies()
+        self.ripprocess = ripprocess()
+        self.tool = tool()
+        self.LOGA_PATH = self.tool.paths()["LOGA_PATH"]
+        self.bin = self.tool.bin()
+        self.aria2c_exe = self.bin["aria2c"]
+        self.last_message_printed = 0
+        self.speed_radar = "0kbps"
+
+    def aria2DisableProxies(self):
+        env = os.environ.copy()
+
+        if env.get("http_proxy"):
+            del env["http_proxy"]
+
+        if env.get("HTTP_PROXY"):
+            del env["HTTP_PROXY"]
+
+        if env.get("https_proxy"):
+            del env["https_proxy"]
+
+        if env.get("HTTPS_PROXY"):
+            del env["HTTPS_PROXY"]
+
+        return env
+
+    def read_stdout(self, line):
+        speed = re.search(r"DL:(.+?)ETA", line)
+        eta = re.search(r"ETA:(.+?)]", line)
+        connection = re.search(r"CN:(.+?)DL", line)
+        percent = re.search(r"\((.*?)\)", line)
+        size = re.search(r" (.*?)/(.*?)\(", line)
+
+        if speed and eta and connection and percent and size:
+            percent = percent.group().strip().replace(")", "").replace("(", "")
+            size = size.group().strip().replace(")", "").replace("(", "")
+            complete, total = size.split("/")
+            connection = connection.group(1).strip()
+            eta = eta.group(1).strip()
+            speed = speed.group(1).strip()
+            self.speed_radar = speed
+
+            stdout_data = {
+                "percent": str(percent),
+                "size": str(total),
+                "complete": str(complete),
+                "total": str(total),
+                "connection": str(connection),
+                "eta": str(eta),
+                "speed": str(speed),
+            }
+
+            return stdout_data
+
+        return None
+
+    def if_errors(self, line):
+        if "exception" in str(line).lower() or "errorcode" in str(line).lower():
+            return line
+        return None
+
+    def delete_last_message_printed(self):
+        print(" " * len(str(self.last_message_printed)), end="\r")
+
+    def get_status(self, stdout_data: dict):
+        return "Aria2c_Status; Size: {Size} | Speed: {Speed} | ETA: {ETA} | Progress: {Complete} -> {Total} ({Percent})".format(
+            Size=stdout_data.get("size"),
+            Speed=stdout_data.get("speed"),
+            ETA=stdout_data.get("eta"),
+            Complete=stdout_data.get("complete"),
+            Total=stdout_data.get("total"),
+            Percent=stdout_data.get("percent"),
+        )
+
+    def is_download_completed(self, line):
+        if "(ok):download completed." in str(line).lower():
+            return "Download completed: (OK) ({}\\s)".format(self.speed_radar)
+        return None
+
+    def start_download(self):
+        proc = subprocess.Popen(
+            self.aria2_download_command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            bufsize=1,
+            universal_newlines=True,
+            env=self.env,
+        )
+
+        check_errors = True
+        for line in getattr(proc, "stdout"):
+            if check_errors:
+                if self.if_errors(line):
+                    raise aria2Error("Aria2c Error {}".format(self.if_errors(line)))
+            check_errors = False
+            stdout_data = self.read_stdout(line)
+            if stdout_data:
+                status_text = self.get_status(stdout_data)
+                self.delete_last_message_printed()
+                print(status_text, end="\r", flush=True)
+                self.last_message_printed = status_text
+            else:
+                download_finished = self.is_download_completed(line)
+                if download_finished:
+                    self.delete_last_message_printed()
+                    print(download_finished, end="\r", flush=True)
+                    self.last_message_printed = download_finished
+        self.logger.info("")
+        return
+
+
+class aria2:
+    def __init__(self,):
+        self.env = self.aria2DisableProxies()
+        self.ripprocess = ripprocess()
+        self.tool = tool()
+        self.bin = self.tool.bin()
+        self.LOGA_PATH = self.tool.paths()["LOGA_PATH"]
+        self.config = self.tool.aria2c()
+        self.aria2c_exe = self.bin["aria2c"]
+        self.logger = logging.getLogger(__name__)
+
+    def convert_args(self, arg):
+        if arg is True:
+            return "true"
+        elif arg is False:
+            return "false"
+        elif arg is None:
+            return "none"
+        else:
+            return str(arg)
+
+    def append_commands(self, command, option_define, option):
+        if option == "skip":
+            return []
+
+        return ["{}{}".format(option_define, option)]
+
+    def append_two_commands(self, command, cmd1, cmd2):
+        if cmd2 == "skip":
+            return []
+
+        return [cmd1] + [cmd2]
+
+    def aria2Options(
+        self,
+        allow_overwrite=True,
+        file_allocation=None,
+        auto_file_renaming=False,
+        async_dns=False,
+        retry_wait=5,
+        summary_interval=0,
+        enable_color=False,
+        connection=16,
+        concurrent_downloads=16,
+        split=16,
+        header="skip",
+        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36",
+        uri_selector="inorder",
+        console_log_level="skip",
+        download_result="hide",
+        quiet="false",
+        extra_commands=[],
+    ):
+
+        options = [] + extra_commands
+        allow_overwrite = self.convert_args(allow_overwrite)
+        quiet = self.convert_args(quiet)
+        file_allocation = self.convert_args(file_allocation)
+        auto_file_renaming = self.convert_args(auto_file_renaming)
+        async_dns = self.convert_args(async_dns)
+        retry_wait = self.convert_args(retry_wait)
+        enable_color = self.convert_args(enable_color)
+        connection = self.convert_args(connection)
+        concurrent_downloads = self.convert_args(concurrent_downloads)
+        split = self.convert_args(split)
+        header = self.convert_args(header)
+        uri_selector = self.convert_args(uri_selector)
+        console_log_level = self.convert_args(console_log_level)
+        download_result = self.convert_args(download_result)
+
+        ##############################################################################
+
+        options += self.append_commands(options, "--allow-overwrite=", allow_overwrite)
+        options += self.append_commands(options, "--quiet=", quiet)
+        options += self.append_commands(options, "--file-allocation=", file_allocation)
+        options += self.append_commands(
+            options, "--auto-file-renaming=", auto_file_renaming
+        )
+        options += self.append_commands(options, "--async-dns=", async_dns)
+        options += self.append_commands(options, "--retry-wait=", retry_wait)
+        options += self.append_commands(options, "--enable-color=", enable_color)
+
+        options += self.append_commands(
+            options, "--max-connection-per-server=", connection
+        )
+
+        options += self.append_commands(
+            options, "--max-concurrent-downloads=", concurrent_downloads
+        )
+        options += self.append_commands(options, "--split=", split)
+        options += self.append_commands(options, "--header=", header)
+        options += self.append_commands(options, "--uri-selector=", uri_selector)
+        options += self.append_commands(
+            options, "--console-log-level=", console_log_level
+        )
+        options += self.append_commands(options, "--download-result=", download_result)
+
+        return options
+
+    def aria2DisableProxies(self):
+        env = os.environ.copy()
+
+        if env.get("http_proxy"):
+            del env["http_proxy"]
+
+        if env.get("HTTP_PROXY"):
+            del env["HTTP_PROXY"]
+
+        if env.get("https_proxy"):
+            del env["https_proxy"]
+
+        if env.get("HTTPS_PROXY"):
+            del env["HTTPS_PROXY"]
+
+        return env
+
+    def aria2DownloadUrl(self, url, output, options, debug=False, moded=False):
+        self.debug = debug
+        aria2_download_command = [self.aria2c_exe] + options
+
+        if self.config["enable_logging"]:
+            LogFile = os.path.join(self.LOGA_PATH, output.replace(".mp4", ".log"))
+            if os.path.isfile(LogFile):
+                os.remove(LogFile)
+            aria2_download_command.append("--log={}".format(LogFile))
+
+        if not url.startswith("http"):
+            raise aria2Error("Url does not start with http/https: {}".format(url))
+
+        aria2_download_command.append(url)
+        aria2_download_command += self.append_two_commands(
+            aria2_download_command, "-o", output
+        )
+
+        self.aria2Debug("Sending Commands to aria2c...")
+        self.aria2Debug(aria2_download_command)
+        self.logger.debug("aria2_download_command: {}".format(aria2_download_command))
+
+        if moded:
+            aria2_moded_download = aria2_moded(aria2_download_command)
+            aria2_moded_download.start_download()
+        else:
+            try:
+                aria = subprocess.call(aria2_download_command, env=self.env)
+            except FileNotFoundError:
+                self.logger.info("UNABLE TO FIND {}".format(self.aria2c_exe))
+                exit(-1)
+            if aria != 0:
+                raise aria2Error("Aria2c exited with code {}".format(aria))
+
+        return
+
+    def aria2DownloadDash(
+        self, segments, output, options, debug=False, moded=False, fixbytes=False
+    ):
+        self.debug = debug
+        aria2_download_command = [self.aria2c_exe] + options
+
+        if self.config["enable_logging"]:
+            LogFile = os.path.join(self.LOGA_PATH, output.replace(".mp4", ".log"))
+            if os.path.isfile(LogFile):
+                os.remove(LogFile)
+            aria2_download_command.append("--log={}".format(LogFile))
+
+        if not isinstance(segments, list) or segments == []:
+            raise aria2Error("invalid list of urls: {}".format(segments))
+
+        if moded:
+            raise aria2Error("moded version not supported for dash downloads atm...")
+
+        txt = output.replace(".mp4", ".txt")
+        folder = output.replace(".mp4", "")
+        segments = list(dict.fromkeys(segments))
+
+        if os.path.exists(folder):
+            shutil.rmtree(folder)
+        if not os.path.exists(folder):
+            os.makedirs(folder)
+
+        segments_location = []
+
+        opened_txt = open(txt, "w+")
+        for num, url in enumerate(segments, start=1):
+            segment_name = str(num).zfill(5) + ".mp4"
+            segments_location.append(os.path.join(*[os.getcwd(), folder, segment_name]))
+            opened_txt.write(url + f"\n out={segment_name}" + f"\n dir={folder}" + "\n")
+        opened_txt.close()
+
+        aria2_download_command += self.append_commands(
+            aria2_download_command, "--input-file=", txt
+        )
+
+        try:
+            aria = subprocess.call(aria2_download_command, env=self.env)
+        except FileNotFoundError:
+            self.logger.info("UNABLE TO FIND {}".format(self.aria2c_exe))
+            exit(-1)
+        if aria != 0:
+            raise aria2Error("Aria2c exited with code {}".format(aria))
+
+        self.logger.info("\nJoining files...")
+        openfile = open(output, "wb")
+        total = int(len(segments_location))
+        for current, fragment in enumerate(segments_location):
+            if os.path.isfile(fragment):
+                if fixbytes:
+                    with open(fragment, "rb") as f:
+                        wvdll = f.read()
+                    if (
+                        re.search(
+                            b"tfhd\x00\x02\x00\x1a\x00\x00\x00\x01\x00\x00\x00\x02",
+                            wvdll,
+                            re.MULTILINE | re.DOTALL,
+                        )
+                        is not None
+                    ):
+                        fw = open(fragment, "wb")
+                        m = re.search(
+                            b"tfhd\x00\x02\x00\x1a\x00\x00\x00\x01\x00\x00\x00",
+                            wvdll,
+                            re.MULTILINE | re.DOTALL,
+                        )
+                        segment_fixed = (
+                            wvdll[: m.end()] + b"\x01" + wvdll[m.end() + 1 :]
+                        )
+                        fw.write(segment_fixed)
+                        fw.close()
+                shutil.copyfileobj(open(fragment, "rb"), openfile)
+                os.remove(fragment)
+                self.ripprocess.updt(total, current + 1)
+        openfile.close()
+
+        if os.path.isfile(txt):
+            os.remove(txt)
+        if os.path.exists(folder):
+            shutil.rmtree(folder)
+
+    def aria2Debug(self, txt):
+        if self.debug:
+            self.logger.info(txt)