Compare commits
23 Commits
d59258cb95
...
bc3745c738
Author | SHA1 | Date |
---|---|---|
Zil0 | bc3745c738 | |
dirkf | 668332b973 | |
dirkf | 0b2ce3685e | |
dirkf | c2766cb80e | |
dirkf | eb38665438 | |
dirkf | e0727e4ab6 | |
Ori Avtalion | 4ea59c6107 | |
dirkf | 21792b88b7 | |
dirkf | d8f134a664 | |
dirkf | 31a15a7c8d | |
dirkf | 19dc10b986 | |
dirkf | 182f63e82a | |
gy-chen | 71211e7db7 | |
Zizheng Guo | a96a45b2cd | |
hatsomatt | 820fae3b3a | |
dirkf | aef24d97e9 | |
dirkf | f7b30e3f73 | |
dirkf | f66372403f | |
dirkf | 7216fa2ac4 | |
dirkf | acc383b9e3 | |
Hubert Hirtz | f0812d7848 | |
Zil0 | 0948f25c26 | |
Zil0 | c5a0627b96 |
|
@ -10,6 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
from test.helper import (
|
from test.helper import (
|
||||||
assertGreaterEqual,
|
assertGreaterEqual,
|
||||||
|
assertLessEqual,
|
||||||
expect_warnings,
|
expect_warnings,
|
||||||
get_params,
|
get_params,
|
||||||
gettestcases,
|
gettestcases,
|
||||||
|
@ -122,7 +123,10 @@ def generator(test_case, tname):
|
||||||
params['outtmpl'] = tname + '_' + params['outtmpl']
|
params['outtmpl'] = tname + '_' + params['outtmpl']
|
||||||
if is_playlist and 'playlist' not in test_case:
|
if is_playlist and 'playlist' not in test_case:
|
||||||
params.setdefault('extract_flat', 'in_playlist')
|
params.setdefault('extract_flat', 'in_playlist')
|
||||||
params.setdefault('playlistend', test_case.get('playlist_mincount'))
|
params.setdefault('playlistend',
|
||||||
|
test_case['playlist_maxcount'] + 1
|
||||||
|
if test_case.get('playlist_maxcount')
|
||||||
|
else test_case.get('playlist_mincount'))
|
||||||
params.setdefault('skip_download', True)
|
params.setdefault('skip_download', True)
|
||||||
|
|
||||||
ydl = YoutubeDL(params, auto_init=False)
|
ydl = YoutubeDL(params, auto_init=False)
|
||||||
|
@ -190,6 +194,14 @@ def generator(test_case, tname):
|
||||||
'Expected at least %d in playlist %s, but got only %d' % (
|
'Expected at least %d in playlist %s, but got only %d' % (
|
||||||
test_case['playlist_mincount'], test_case['url'],
|
test_case['playlist_mincount'], test_case['url'],
|
||||||
len(res_dict['entries'])))
|
len(res_dict['entries'])))
|
||||||
|
if 'playlist_maxcount' in test_case:
|
||||||
|
assertLessEqual(
|
||||||
|
self,
|
||||||
|
len(res_dict['entries']),
|
||||||
|
test_case['playlist_maxcount'],
|
||||||
|
'Expected at most %d in playlist %s, but got %d' % (
|
||||||
|
test_case['playlist_maxcount'], test_case['url'],
|
||||||
|
len(res_dict['entries'])))
|
||||||
if 'playlist_count' in test_case:
|
if 'playlist_count' in test_case:
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
len(res_dict['entries']),
|
len(res_dict['entries']),
|
||||||
|
|
|
@ -18,6 +18,7 @@ from test.helper import (
|
||||||
)
|
)
|
||||||
from youtube_dl import YoutubeDL
|
from youtube_dl import YoutubeDL
|
||||||
from youtube_dl.compat import (
|
from youtube_dl.compat import (
|
||||||
|
compat_contextlib_suppress,
|
||||||
compat_http_cookiejar_Cookie,
|
compat_http_cookiejar_Cookie,
|
||||||
compat_http_server,
|
compat_http_server,
|
||||||
compat_kwargs,
|
compat_kwargs,
|
||||||
|
@ -35,6 +36,9 @@ from youtube_dl.downloader.external import (
|
||||||
HttpieFD,
|
HttpieFD,
|
||||||
WgetFD,
|
WgetFD,
|
||||||
)
|
)
|
||||||
|
from youtube_dl.postprocessor import (
|
||||||
|
FFmpegPostProcessor,
|
||||||
|
)
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
TEST_SIZE = 10 * 1024
|
TEST_SIZE = 10 * 1024
|
||||||
|
@ -227,7 +231,17 @@ class TestAria2cFD(unittest.TestCase):
|
||||||
self.assertIn('--load-cookies=%s' % downloader._cookies_tempfile, cmd)
|
self.assertIn('--load-cookies=%s' % downloader._cookies_tempfile, cmd)
|
||||||
|
|
||||||
|
|
||||||
@ifExternalFDAvailable(FFmpegFD)
|
# Handle delegated availability
|
||||||
|
def ifFFmpegFDAvailable(externalFD):
|
||||||
|
# raise SkipTest, or set False!
|
||||||
|
avail = ifExternalFDAvailable(externalFD) and False
|
||||||
|
with compat_contextlib_suppress(Exception):
|
||||||
|
avail = FFmpegPostProcessor(downloader=None).available
|
||||||
|
return unittest.skipUnless(
|
||||||
|
avail, externalFD.get_basename() + ' not found')
|
||||||
|
|
||||||
|
|
||||||
|
@ifFFmpegFDAvailable(FFmpegFD)
|
||||||
class TestFFmpegFD(unittest.TestCase):
|
class TestFFmpegFD(unittest.TestCase):
|
||||||
_args = []
|
_args = []
|
||||||
|
|
||||||
|
|
|
@ -81,6 +81,7 @@ from youtube_dl.utils import (
|
||||||
sanitize_filename,
|
sanitize_filename,
|
||||||
sanitize_path,
|
sanitize_path,
|
||||||
sanitize_url,
|
sanitize_url,
|
||||||
|
sanitized_Request,
|
||||||
shell_quote,
|
shell_quote,
|
||||||
smuggle_url,
|
smuggle_url,
|
||||||
str_or_none,
|
str_or_none,
|
||||||
|
@ -255,6 +256,18 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertEqual(sanitize_url('https://foo.bar'), 'https://foo.bar')
|
self.assertEqual(sanitize_url('https://foo.bar'), 'https://foo.bar')
|
||||||
self.assertEqual(sanitize_url('foo bar'), 'foo bar')
|
self.assertEqual(sanitize_url('foo bar'), 'foo bar')
|
||||||
|
|
||||||
|
def test_sanitized_Request(self):
|
||||||
|
self.assertFalse(sanitized_Request('http://foo.bar').has_header('Authorization'))
|
||||||
|
self.assertFalse(sanitized_Request('http://:foo.bar').has_header('Authorization'))
|
||||||
|
self.assertEqual(sanitized_Request('http://@foo.bar').get_header('Authorization'),
|
||||||
|
'Basic Og==')
|
||||||
|
self.assertEqual(sanitized_Request('http://:pass@foo.bar').get_header('Authorization'),
|
||||||
|
'Basic OnBhc3M=')
|
||||||
|
self.assertEqual(sanitized_Request('http://user:@foo.bar').get_header('Authorization'),
|
||||||
|
'Basic dXNlcjo=')
|
||||||
|
self.assertEqual(sanitized_Request('http://user:pass@foo.bar').get_header('Authorization'),
|
||||||
|
'Basic dXNlcjpwYXNz')
|
||||||
|
|
||||||
def test_expand_path(self):
|
def test_expand_path(self):
|
||||||
def env(var):
|
def env(var):
|
||||||
return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var)
|
return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var)
|
||||||
|
@ -512,11 +525,14 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertEqual(float_or_none(set()), None)
|
self.assertEqual(float_or_none(set()), None)
|
||||||
|
|
||||||
def test_int_or_none(self):
|
def test_int_or_none(self):
|
||||||
|
self.assertEqual(int_or_none(42), 42)
|
||||||
self.assertEqual(int_or_none('42'), 42)
|
self.assertEqual(int_or_none('42'), 42)
|
||||||
self.assertEqual(int_or_none(''), None)
|
self.assertEqual(int_or_none(''), None)
|
||||||
self.assertEqual(int_or_none(None), None)
|
self.assertEqual(int_or_none(None), None)
|
||||||
self.assertEqual(int_or_none([]), None)
|
self.assertEqual(int_or_none([]), None)
|
||||||
self.assertEqual(int_or_none(set()), None)
|
self.assertEqual(int_or_none(set()), None)
|
||||||
|
self.assertEqual(int_or_none('42', base=8), 34)
|
||||||
|
self.assertRaises(TypeError, int_or_none(42, base=8))
|
||||||
|
|
||||||
def test_str_to_int(self):
|
def test_str_to_int(self):
|
||||||
self.assertEqual(str_to_int('123,456'), 123456)
|
self.assertEqual(str_to_int('123,456'), 123456)
|
||||||
|
|
|
@ -1829,10 +1829,13 @@ class YoutubeDL(object):
|
||||||
if download:
|
if download:
|
||||||
if len(formats_to_download) > 1:
|
if len(formats_to_download) > 1:
|
||||||
self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
|
self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
|
||||||
|
filenames = []
|
||||||
for format in formats_to_download:
|
for format in formats_to_download:
|
||||||
new_info = dict(info_dict)
|
new_info = dict(info_dict)
|
||||||
new_info.update(format)
|
new_info.update(format)
|
||||||
self.process_info(new_info)
|
self.process_info(new_info)
|
||||||
|
filenames.append(new_info.get('_filename'))
|
||||||
|
info_dict['filenames'] = filenames
|
||||||
# We update the info dict with the best quality format (backwards compatibility)
|
# We update the info dict with the best quality format (backwards compatibility)
|
||||||
info_dict.update(formats_to_download[-1])
|
info_dict.update(formats_to_download[-1])
|
||||||
return info_dict
|
return info_dict
|
||||||
|
@ -2099,7 +2102,7 @@ class YoutubeDL(object):
|
||||||
self.report_warning(
|
self.report_warning(
|
||||||
'Requested formats are incompatible for merge and will be merged into mkv.')
|
'Requested formats are incompatible for merge and will be merged into mkv.')
|
||||||
# Ensure filename always has a correct extension for successful merge
|
# Ensure filename always has a correct extension for successful merge
|
||||||
filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
|
info_dict['_filename'] = filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
|
||||||
if os.path.exists(encodeFilename(filename)):
|
if os.path.exists(encodeFilename(filename)):
|
||||||
self.to_screen(
|
self.to_screen(
|
||||||
'[download] %s has already been downloaded and '
|
'[download] %s has already been downloaded and '
|
||||||
|
@ -2282,19 +2285,26 @@ class YoutubeDL(object):
|
||||||
if ie_info.get('__postprocessors') is not None:
|
if ie_info.get('__postprocessors') is not None:
|
||||||
pps_chain.extend(ie_info['__postprocessors'])
|
pps_chain.extend(ie_info['__postprocessors'])
|
||||||
pps_chain.extend(self._pps)
|
pps_chain.extend(self._pps)
|
||||||
|
filepaths = []
|
||||||
for pp in pps_chain:
|
for pp in pps_chain:
|
||||||
files_to_delete = []
|
files_to_delete = []
|
||||||
try:
|
try:
|
||||||
files_to_delete, info = pp.run(info)
|
files_to_delete, info = pp.run(info)
|
||||||
except PostProcessingError as e:
|
except PostProcessingError as e:
|
||||||
self.report_error(e.msg)
|
self.report_error(e.msg)
|
||||||
if files_to_delete and not self.params.get('keepvideo', False):
|
if files_to_delete:
|
||||||
for old_filename in files_to_delete:
|
if self.params.get('keepvideo'):
|
||||||
self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
|
filepaths.extend(files_to_delete)
|
||||||
try:
|
else:
|
||||||
os.remove(encodeFilename(old_filename))
|
for old_filename in files_to_delete:
|
||||||
except (IOError, OSError):
|
self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
|
||||||
self.report_warning('Unable to remove downloaded original file')
|
try:
|
||||||
|
os.remove(encodeFilename(old_filename))
|
||||||
|
except (IOError, OSError):
|
||||||
|
self.report_warning('Unable to remove downloaded original file')
|
||||||
|
if info.get('filepath'):
|
||||||
|
filepaths.append(info['filepath'])
|
||||||
|
ie_info['_filename'] = filepaths or filename
|
||||||
|
|
||||||
def _make_archive_id(self, info_dict):
|
def _make_archive_id(self, info_dict):
|
||||||
video_id = info_dict.get('id')
|
video_id = info_dict.get('id')
|
||||||
|
|
|
@ -2421,29 +2421,26 @@ except ImportError: # Python 2
|
||||||
compat_urllib_request_urlretrieve = compat_urlretrieve
|
compat_urllib_request_urlretrieve = compat_urlretrieve
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
from HTMLParser import (
|
||||||
|
HTMLParser as compat_HTMLParser,
|
||||||
|
HTMLParseError as compat_HTMLParseError)
|
||||||
|
except ImportError: # Python 3
|
||||||
from html.parser import HTMLParser as compat_HTMLParser
|
from html.parser import HTMLParser as compat_HTMLParser
|
||||||
except ImportError: # Python 2
|
|
||||||
from HTMLParser import HTMLParser as compat_HTMLParser
|
|
||||||
compat_html_parser_HTMLParser = compat_HTMLParser
|
|
||||||
|
|
||||||
try: # Python 2
|
|
||||||
from HTMLParser import HTMLParseError as compat_HTMLParseError
|
|
||||||
except ImportError: # Python <3.4
|
|
||||||
try:
|
try:
|
||||||
from html.parser import HTMLParseError as compat_HTMLParseError
|
from html.parser import HTMLParseError as compat_HTMLParseError
|
||||||
except ImportError: # Python >3.4
|
except ImportError: # Python >3.4
|
||||||
|
# HTMLParseError was deprecated in Python 3.3 and removed in
|
||||||
# HTMLParseError has been deprecated in Python 3.3 and removed in
|
|
||||||
# Python 3.5. Introducing dummy exception for Python >3.5 for compatible
|
# Python 3.5. Introducing dummy exception for Python >3.5 for compatible
|
||||||
# and uniform cross-version exception handling
|
# and uniform cross-version exception handling
|
||||||
class compat_HTMLParseError(Exception):
|
class compat_HTMLParseError(Exception):
|
||||||
pass
|
pass
|
||||||
|
compat_html_parser_HTMLParser = compat_HTMLParser
|
||||||
compat_html_parser_HTMLParseError = compat_HTMLParseError
|
compat_html_parser_HTMLParseError = compat_HTMLParseError
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from subprocess import DEVNULL
|
_DEVNULL = subprocess.DEVNULL
|
||||||
compat_subprocess_get_DEVNULL = lambda: DEVNULL
|
compat_subprocess_get_DEVNULL = lambda: _DEVNULL
|
||||||
except ImportError:
|
except AttributeError:
|
||||||
compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
|
compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -2943,6 +2940,51 @@ else:
|
||||||
compat_socket_create_connection = socket.create_connection
|
compat_socket_create_connection = socket.create_connection
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
from contextlib import suppress as compat_contextlib_suppress
|
||||||
|
except ImportError:
|
||||||
|
class compat_contextlib_suppress(object):
|
||||||
|
_exceptions = None
|
||||||
|
|
||||||
|
def __init__(self, *exceptions):
|
||||||
|
super(compat_contextlib_suppress, self).__init__()
|
||||||
|
# TODO: [Base]ExceptionGroup (3.12+)
|
||||||
|
self._exceptions = exceptions
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
|
return exc_val is not None and isinstance(exc_val, self._exceptions or tuple())
|
||||||
|
|
||||||
|
|
||||||
|
# subprocess.Popen context manager
|
||||||
|
# avoids leaking handles if .communicate() is not called
|
||||||
|
try:
|
||||||
|
_Popen = subprocess.Popen
|
||||||
|
# check for required context manager attributes
|
||||||
|
_Popen.__enter__ and _Popen.__exit__
|
||||||
|
compat_subprocess_Popen = _Popen
|
||||||
|
except AttributeError:
|
||||||
|
# not a context manager - make one
|
||||||
|
from contextlib import contextmanager
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def compat_subprocess_Popen(*args, **kwargs):
|
||||||
|
popen = None
|
||||||
|
try:
|
||||||
|
popen = _Popen(*args, **kwargs)
|
||||||
|
yield popen
|
||||||
|
finally:
|
||||||
|
if popen:
|
||||||
|
for f in (popen.stdin, popen.stdout, popen.stderr):
|
||||||
|
if f:
|
||||||
|
# repeated .close() is OK, but just in case
|
||||||
|
with compat_contextlib_suppress(EnvironmentError):
|
||||||
|
f.close()
|
||||||
|
popen.wait()
|
||||||
|
|
||||||
|
|
||||||
# Fix https://github.com/ytdl-org/youtube-dl/issues/4223
|
# Fix https://github.com/ytdl-org/youtube-dl/issues/4223
|
||||||
# See http://bugs.python.org/issue9161 for what is broken
|
# See http://bugs.python.org/issue9161 for what is broken
|
||||||
def workaround_optparse_bug9161():
|
def workaround_optparse_bug9161():
|
||||||
|
@ -3263,6 +3305,7 @@ __all__ = [
|
||||||
'compat_http_cookiejar_Cookie',
|
'compat_http_cookiejar_Cookie',
|
||||||
'compat_http_cookies',
|
'compat_http_cookies',
|
||||||
'compat_http_cookies_SimpleCookie',
|
'compat_http_cookies_SimpleCookie',
|
||||||
|
'compat_contextlib_suppress',
|
||||||
'compat_ctypes_WINFUNCTYPE',
|
'compat_ctypes_WINFUNCTYPE',
|
||||||
'compat_etree_fromstring',
|
'compat_etree_fromstring',
|
||||||
'compat_filter',
|
'compat_filter',
|
||||||
|
@ -3298,6 +3341,7 @@ __all__ = [
|
||||||
'compat_struct_pack',
|
'compat_struct_pack',
|
||||||
'compat_struct_unpack',
|
'compat_struct_unpack',
|
||||||
'compat_subprocess_get_DEVNULL',
|
'compat_subprocess_get_DEVNULL',
|
||||||
|
'compat_subprocess_Popen',
|
||||||
'compat_tokenize_tokenize',
|
'compat_tokenize_tokenize',
|
||||||
'compat_urllib_error',
|
'compat_urllib_error',
|
||||||
'compat_urllib_parse',
|
'compat_urllib_parse',
|
||||||
|
|
|
@ -11,8 +11,14 @@ from .common import FileDownloader
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_setenv,
|
compat_setenv,
|
||||||
compat_str,
|
compat_str,
|
||||||
|
compat_subprocess_Popen,
|
||||||
)
|
)
|
||||||
from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
|
|
||||||
|
try:
|
||||||
|
from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
|
||||||
|
except ImportError:
|
||||||
|
FFmpegPostProcessor = None
|
||||||
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
cli_option,
|
cli_option,
|
||||||
cli_valueless_option,
|
cli_valueless_option,
|
||||||
|
@ -361,13 +367,14 @@ class FFmpegFD(ExternalFD):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def available(cls):
|
def available(cls):
|
||||||
return FFmpegPostProcessor().available
|
# actual availability can only be confirmed for an instance
|
||||||
|
return bool(FFmpegPostProcessor)
|
||||||
|
|
||||||
def _call_downloader(self, tmpfilename, info_dict):
|
def _call_downloader(self, tmpfilename, info_dict):
|
||||||
url = info_dict['url']
|
# `downloader` means the parent `YoutubeDL`
|
||||||
ffpp = FFmpegPostProcessor(downloader=self)
|
ffpp = FFmpegPostProcessor(downloader=self.ydl)
|
||||||
if not ffpp.available:
|
if not ffpp.available:
|
||||||
self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
|
self.report_error('ffmpeg required for download but no ffmpeg (nor avconv) executable could be found. Please install one.')
|
||||||
return False
|
return False
|
||||||
ffpp.check_version()
|
ffpp.check_version()
|
||||||
|
|
||||||
|
@ -396,6 +403,7 @@ class FFmpegFD(ExternalFD):
|
||||||
# if end_time:
|
# if end_time:
|
||||||
# args += ['-t', compat_str(end_time - start_time)]
|
# args += ['-t', compat_str(end_time - start_time)]
|
||||||
|
|
||||||
|
url = info_dict['url']
|
||||||
cookies = self.ydl.cookiejar.get_cookies_for_url(url)
|
cookies = self.ydl.cookiejar.get_cookies_for_url(url)
|
||||||
if cookies:
|
if cookies:
|
||||||
args.extend(['-cookies', ''.join(
|
args.extend(['-cookies', ''.join(
|
||||||
|
@ -483,21 +491,25 @@ class FFmpegFD(ExternalFD):
|
||||||
|
|
||||||
self._debug_cmd(args)
|
self._debug_cmd(args)
|
||||||
|
|
||||||
proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
|
# From [1], a PIPE opened in Popen() should be closed, unless
|
||||||
try:
|
# .communicate() is called. Avoid leaking any PIPEs by using Popen
|
||||||
retval = proc.wait()
|
# as a context manager (newer Python 3.x and compat)
|
||||||
except BaseException as e:
|
# Fixes "Resource Warning" in test/test_downloader_external.py
|
||||||
# subprocess.run would send the SIGKILL signal to ffmpeg and the
|
# [1] https://devpress.csdn.net/python/62fde12d7e66823466192e48.html
|
||||||
# mp4 file couldn't be played, but if we ask ffmpeg to quit it
|
with compat_subprocess_Popen(args, stdin=subprocess.PIPE, env=env) as proc:
|
||||||
# produces a file that is playable (this is mostly useful for live
|
try:
|
||||||
# streams). Note that Windows is not affected and produces playable
|
retval = proc.wait()
|
||||||
# files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
|
except BaseException as e:
|
||||||
if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
|
# subprocess.run would send the SIGKILL signal to ffmpeg and the
|
||||||
process_communicate_or_kill(proc, b'q')
|
# mp4 file couldn't be played, but if we ask ffmpeg to quit it
|
||||||
else:
|
# produces a file that is playable (this is mostly useful for live
|
||||||
proc.kill()
|
# streams). Note that Windows is not affected and produces playable
|
||||||
proc.wait()
|
# files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
|
||||||
raise
|
if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
|
||||||
|
process_communicate_or_kill(proc, b'q')
|
||||||
|
else:
|
||||||
|
proc.kill()
|
||||||
|
raise
|
||||||
return retval
|
return retval
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -25,6 +25,7 @@ from ..compat import (
|
||||||
compat_getpass,
|
compat_getpass,
|
||||||
compat_integer_types,
|
compat_integer_types,
|
||||||
compat_http_client,
|
compat_http_client,
|
||||||
|
compat_kwargs,
|
||||||
compat_map as map,
|
compat_map as map,
|
||||||
compat_open as open,
|
compat_open as open,
|
||||||
compat_os_name,
|
compat_os_name,
|
||||||
|
@ -1102,6 +1103,60 @@ class InfoExtractor(object):
|
||||||
self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
|
self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def _search_json(self, start_pattern, string, name, video_id, **kwargs):
|
||||||
|
"""Searches string for the JSON object specified by start_pattern"""
|
||||||
|
|
||||||
|
# self, start_pattern, string, name, video_id, *, end_pattern='',
|
||||||
|
# contains_pattern=r'{(?s:.+)}', fatal=True, default=NO_DEFAULT
|
||||||
|
# NB: end_pattern is only used to reduce the size of the initial match
|
||||||
|
end_pattern = kwargs.pop('end_pattern', '')
|
||||||
|
# (?:[\s\S]) simulates (?(s):.) (eg)
|
||||||
|
contains_pattern = kwargs.pop('contains_pattern', r'{[\s\S]+}')
|
||||||
|
fatal = kwargs.pop('fatal', True)
|
||||||
|
default = kwargs.pop('default', NO_DEFAULT)
|
||||||
|
|
||||||
|
if default is NO_DEFAULT:
|
||||||
|
default, has_default = {}, False
|
||||||
|
else:
|
||||||
|
fatal, has_default = False, True
|
||||||
|
|
||||||
|
json_string = self._search_regex(
|
||||||
|
r'(?:{0})\s*(?P<json>{1})\s*(?:{2})'.format(
|
||||||
|
start_pattern, contains_pattern, end_pattern),
|
||||||
|
string, name, group='json', fatal=fatal, default=None if has_default else NO_DEFAULT)
|
||||||
|
if not json_string:
|
||||||
|
return default
|
||||||
|
|
||||||
|
# yt-dlp has a special JSON parser that allows trailing text.
|
||||||
|
# Until that arrives here, the diagnostic from the exception
|
||||||
|
# raised by json.loads() is used to extract the wanted text.
|
||||||
|
# Either way, it's a problem if a transform_source() can't
|
||||||
|
# handle the trailing text.
|
||||||
|
|
||||||
|
# force an exception
|
||||||
|
kwargs['fatal'] = True
|
||||||
|
|
||||||
|
# self._downloader._format_err(name, self._downloader.Styles.EMPHASIS)
|
||||||
|
for _ in range(2):
|
||||||
|
try:
|
||||||
|
# return self._parse_json(json_string, video_id, ignore_extra=True, **kwargs)
|
||||||
|
transform_source = kwargs.pop('transform_source', None)
|
||||||
|
if transform_source:
|
||||||
|
json_string = transform_source(json_string)
|
||||||
|
return self._parse_json(json_string, video_id, **compat_kwargs(kwargs))
|
||||||
|
except ExtractorError as e:
|
||||||
|
end = int_or_none(self._search_regex(r'\(char\s+(\d+)', error_to_compat_str(e), 'end', default=None))
|
||||||
|
if end is not None:
|
||||||
|
json_string = json_string[:end]
|
||||||
|
continue
|
||||||
|
msg = 'Unable to extract {0} - Failed to parse JSON'.format(name)
|
||||||
|
if fatal:
|
||||||
|
raise ExtractorError(msg, cause=e.cause, video_id=video_id)
|
||||||
|
elif not has_default:
|
||||||
|
self.report_warning(
|
||||||
|
'{0}: {1}'.format(msg, error_to_compat_str(e)), video_id=video_id)
|
||||||
|
return default
|
||||||
|
|
||||||
def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
|
def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
|
||||||
"""
|
"""
|
||||||
Like _search_regex, but strips HTML tags and unescapes entities.
|
Like _search_regex, but strips HTML tags and unescapes entities.
|
||||||
|
@ -2966,25 +3021,22 @@ class InfoExtractor(object):
|
||||||
return formats
|
return formats
|
||||||
|
|
||||||
def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
|
def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
|
||||||
mobj = re.search(
|
return self._search_json(
|
||||||
r'''(?s)jwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?!</script>).*?\.\s*setup\s*\(\s*(?P<options>(?:\([^)]*\)|[^)])+)\s*\)''',
|
r'''(?<!-)\bjwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?:(?!</script>).)*?\.\s*(?:setup\s*\(|(?P<load>load)\s*\(\s*\[)''',
|
||||||
webpage)
|
webpage, 'JWPlayer data', video_id,
|
||||||
if mobj:
|
# must be a {...} or sequence, ending
|
||||||
try:
|
contains_pattern=r'\{[\s\S]*}(?(load)(?:\s*,\s*\{[\s\S]*})*)', end_pattern=r'(?(load)\]|\))',
|
||||||
jwplayer_data = self._parse_json(mobj.group('options'),
|
transform_source=transform_source, default=None)
|
||||||
video_id=video_id,
|
|
||||||
transform_source=transform_source)
|
|
||||||
except ExtractorError:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
if isinstance(jwplayer_data, dict):
|
|
||||||
return jwplayer_data
|
|
||||||
|
|
||||||
def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
|
def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
|
||||||
jwplayer_data = self._find_jwplayer_data(
|
|
||||||
webpage, video_id, transform_source=js_to_json)
|
# allow passing `transform_source` through to _find_jwplayer_data()
|
||||||
return self._parse_jwplayer_data(
|
transform_source = kwargs.pop('transform_source', None)
|
||||||
jwplayer_data, video_id, *args, **kwargs)
|
kwfind = compat_kwargs({'transform_source': transform_source}) if transform_source else {}
|
||||||
|
|
||||||
|
jwplayer_data = self._find_jwplayer_data(webpage, video_id, **kwfind)
|
||||||
|
|
||||||
|
return self._parse_jwplayer_data(jwplayer_data, video_id, *args, **kwargs)
|
||||||
|
|
||||||
def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
|
def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
|
||||||
m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
|
m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
|
||||||
|
@ -3018,22 +3070,14 @@ class InfoExtractor(object):
|
||||||
mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
|
mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
|
||||||
|
|
||||||
subtitles = {}
|
subtitles = {}
|
||||||
tracks = video_data.get('tracks')
|
for track in traverse_obj(video_data, (
|
||||||
if tracks and isinstance(tracks, list):
|
'tracks', lambda _, t: t.get('kind').lower() in ('captions', 'subtitles'))):
|
||||||
for track in tracks:
|
track_url = urljoin(base_url, track.get('file'))
|
||||||
if not isinstance(track, dict):
|
if not track_url:
|
||||||
continue
|
continue
|
||||||
track_kind = track.get('kind')
|
subtitles.setdefault(track.get('label') or 'en', []).append({
|
||||||
if not track_kind or not isinstance(track_kind, compat_str):
|
'url': self._proto_relative_url(track_url)
|
||||||
continue
|
})
|
||||||
if track_kind.lower() not in ('captions', 'subtitles'):
|
|
||||||
continue
|
|
||||||
track_url = urljoin(base_url, track.get('file'))
|
|
||||||
if not track_url:
|
|
||||||
continue
|
|
||||||
subtitles.setdefault(track.get('label') or 'en', []).append({
|
|
||||||
'url': self._proto_relative_url(track_url)
|
|
||||||
})
|
|
||||||
|
|
||||||
entry = {
|
entry = {
|
||||||
'id': this_video_id,
|
'id': this_video_id,
|
||||||
|
|
|
@ -383,7 +383,6 @@ from .fc2 import (
|
||||||
FC2EmbedIE,
|
FC2EmbedIE,
|
||||||
)
|
)
|
||||||
from .fczenit import FczenitIE
|
from .fczenit import FczenitIE
|
||||||
from .filemoon import FileMoonIE
|
|
||||||
from .fifa import FifaIE
|
from .fifa import FifaIE
|
||||||
from .filmon import (
|
from .filmon import (
|
||||||
FilmOnIE,
|
FilmOnIE,
|
||||||
|
@ -1654,7 +1653,15 @@ from .younow import (
|
||||||
YouNowChannelIE,
|
YouNowChannelIE,
|
||||||
YouNowMomentIE,
|
YouNowMomentIE,
|
||||||
)
|
)
|
||||||
from .youporn import YouPornIE
|
from .youporn import (
|
||||||
|
YouPornIE,
|
||||||
|
YouPornCategoryIE,
|
||||||
|
YouPornChannelIE,
|
||||||
|
YouPornCollectionIE,
|
||||||
|
YouPornStarIE,
|
||||||
|
YouPornTagIE,
|
||||||
|
YouPornVideosIE,
|
||||||
|
)
|
||||||
from .yourporn import YourPornIE
|
from .yourporn import YourPornIE
|
||||||
from .yourupload import YourUploadIE
|
from .yourupload import YourUploadIE
|
||||||
from .youtube import (
|
from .youtube import (
|
||||||
|
|
|
@ -1,43 +0,0 @@
|
||||||
# coding: utf-8
|
|
||||||
from __future__ import unicode_literals
|
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
|
||||||
from ..utils import (
|
|
||||||
decode_packed_codes,
|
|
||||||
js_to_json,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class FileMoonIE(InfoExtractor):
|
|
||||||
_VALID_URL = r'https?://(?:www\.)?filemoon\.sx/./(?P<id>\w+)'
|
|
||||||
_TEST = {
|
|
||||||
'url': 'https://filemoon.sx/e/dw40rxrzruqz',
|
|
||||||
'md5': '5a713742f57ac4aef29b74733e8dda01',
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'dw40rxrzruqz',
|
|
||||||
'title': 'dw40rxrzruqz',
|
|
||||||
'ext': 'mp4'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
|
||||||
video_id = self._match_id(url)
|
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
|
||||||
matches = re.findall(r'(?s)(eval.*?)</script>', webpage)
|
|
||||||
packed = matches[-1]
|
|
||||||
unpacked = decode_packed_codes(packed)
|
|
||||||
jwplayer_sources = self._parse_json(
|
|
||||||
self._search_regex(
|
|
||||||
r'(?s)player\s*\.\s*setup\s*\(\s*\{\s*sources\s*:\s*(.*?])', unpacked, 'jwplayer sources'),
|
|
||||||
video_id, transform_source=js_to_json)
|
|
||||||
|
|
||||||
formats = self._parse_jwplayer_formats(jwplayer_sources, video_id)
|
|
||||||
|
|
||||||
return {
|
|
||||||
'id': video_id,
|
|
||||||
'title': self._generic_title(url) or video_id,
|
|
||||||
'formats': formats
|
|
||||||
}
|
|
|
@ -6,22 +6,31 @@ import re
|
||||||
import string
|
import string
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
|
from ..compat import (
|
||||||
|
compat_b64decode,
|
||||||
|
compat_ord,
|
||||||
|
compat_struct_pack,
|
||||||
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
mimetype2ext,
|
mimetype2ext,
|
||||||
parse_codecs,
|
parse_codecs,
|
||||||
|
parse_qs,
|
||||||
update_url_query,
|
update_url_query,
|
||||||
urljoin,
|
urljoin,
|
||||||
xpath_element,
|
xpath_element,
|
||||||
xpath_text,
|
xpath_text,
|
||||||
)
|
)
|
||||||
from ..compat import (
|
|
||||||
compat_b64decode,
|
|
||||||
compat_ord,
|
def compat_random_choices(population, *args, **kwargs):
|
||||||
compat_struct_pack,
|
# weights=None, *, cum_weights=None, k=1
|
||||||
compat_urlparse,
|
# limited implementation needed here
|
||||||
)
|
weights = args[0] if args else kwargs.get('weights')
|
||||||
|
assert all(w is None for w in (weights, kwargs.get('cum_weights')))
|
||||||
|
k = kwargs.get('k', 1)
|
||||||
|
return ''.join(random.choice(population) for _ in range(k))
|
||||||
|
|
||||||
|
|
||||||
class VideaIE(InfoExtractor):
|
class VideaIE(InfoExtractor):
|
||||||
|
@ -35,6 +44,7 @@ class VideaIE(InfoExtractor):
|
||||||
)
|
)
|
||||||
(?P<id>[^?#&]+)
|
(?P<id>[^?#&]+)
|
||||||
'''
|
'''
|
||||||
|
_EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//videa\.hu/player\?.*?\bv=.+?)\1']
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://videa.hu/videok/allatok/az-orult-kigyasz-285-kigyot-kigyo-8YfIAjxwWGwT8HVQ',
|
'url': 'http://videa.hu/videok/allatok/az-orult-kigyasz-285-kigyot-kigyo-8YfIAjxwWGwT8HVQ',
|
||||||
'md5': '97a7af41faeaffd9f1fc864a7c7e7603',
|
'md5': '97a7af41faeaffd9f1fc864a7c7e7603',
|
||||||
|
@ -44,6 +54,7 @@ class VideaIE(InfoExtractor):
|
||||||
'title': 'Az őrült kígyász 285 kígyót enged szabadon',
|
'title': 'Az őrült kígyász 285 kígyót enged szabadon',
|
||||||
'thumbnail': r're:^https?://.*',
|
'thumbnail': r're:^https?://.*',
|
||||||
'duration': 21,
|
'duration': 21,
|
||||||
|
'age_limit': 0,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://videa.hu/videok/origo/jarmuvek/supercars-elozes-jAHDWfWSJH5XuFhH',
|
'url': 'http://videa.hu/videok/origo/jarmuvek/supercars-elozes-jAHDWfWSJH5XuFhH',
|
||||||
|
@ -54,6 +65,7 @@ class VideaIE(InfoExtractor):
|
||||||
'title': 'Supercars előzés',
|
'title': 'Supercars előzés',
|
||||||
'thumbnail': r're:^https?://.*',
|
'thumbnail': r're:^https?://.*',
|
||||||
'duration': 64,
|
'duration': 64,
|
||||||
|
'age_limit': 0,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://videa.hu/player?v=8YfIAjxwWGwT8HVQ',
|
'url': 'http://videa.hu/player?v=8YfIAjxwWGwT8HVQ',
|
||||||
|
@ -64,6 +76,7 @@ class VideaIE(InfoExtractor):
|
||||||
'title': 'Az őrült kígyász 285 kígyót enged szabadon',
|
'title': 'Az őrült kígyász 285 kígyót enged szabadon',
|
||||||
'thumbnail': r're:^https?://.*',
|
'thumbnail': r're:^https?://.*',
|
||||||
'duration': 21,
|
'duration': 21,
|
||||||
|
'age_limit': 0,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://videa.hu/player/v/8YfIAjxwWGwT8HVQ?autoplay=1',
|
'url': 'http://videa.hu/player/v/8YfIAjxwWGwT8HVQ?autoplay=1',
|
||||||
|
@ -80,11 +93,14 @@ class VideaIE(InfoExtractor):
|
||||||
}]
|
}]
|
||||||
_STATIC_SECRET = 'xHb0ZvME5q8CBcoQi6AngerDu3FGO9fkUlwPmLVY_RTzj2hJIS4NasXWKy1td7p'
|
_STATIC_SECRET = 'xHb0ZvME5q8CBcoQi6AngerDu3FGO9fkUlwPmLVY_RTzj2hJIS4NasXWKy1td7p'
|
||||||
|
|
||||||
@staticmethod
|
@classmethod
|
||||||
def _extract_urls(webpage):
|
def _extract_urls(cls, webpage):
|
||||||
return [url for _, url in re.findall(
|
def yield_urls():
|
||||||
r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//videa\.hu/player\?.*?\bv=.+?)\1',
|
for pattern in cls._EMBED_REGEX:
|
||||||
webpage)]
|
for m in re.finditer(pattern, webpage):
|
||||||
|
yield m.group('url')
|
||||||
|
|
||||||
|
return list(yield_urls())
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def rc4(cipher_text, key):
|
def rc4(cipher_text, key):
|
||||||
|
@ -130,13 +146,13 @@ class VideaIE(InfoExtractor):
|
||||||
for i in range(0, 32):
|
for i in range(0, 32):
|
||||||
result += s[i - (self._STATIC_SECRET.index(l[i]) - 31)]
|
result += s[i - (self._STATIC_SECRET.index(l[i]) - 31)]
|
||||||
|
|
||||||
query = compat_urlparse.parse_qs(compat_urlparse.urlparse(player_url).query)
|
query = parse_qs(player_url)
|
||||||
random_seed = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
|
random_seed = ''.join(compat_random_choices(string.ascii_letters + string.digits, k=8))
|
||||||
query['_s'] = random_seed
|
query['_s'] = random_seed
|
||||||
query['_t'] = result[:16]
|
query['_t'] = result[:16]
|
||||||
|
|
||||||
b64_info, handle = self._download_webpage_handle(
|
b64_info, handle = self._download_webpage_handle(
|
||||||
'http://videa.hu/videaplayer_get_xml.php', video_id, query=query)
|
'http://videa.hu/player/xml', video_id, query=query)
|
||||||
if b64_info.startswith('<?xml'):
|
if b64_info.startswith('<?xml'):
|
||||||
info = self._parse_xml(b64_info, video_id)
|
info = self._parse_xml(b64_info, video_id)
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -673,8 +673,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
|
||||||
raise
|
raise
|
||||||
|
|
||||||
if '//player.vimeo.com/video/' in url:
|
if '//player.vimeo.com/video/' in url:
|
||||||
config = self._parse_json(self._search_regex(
|
config = self._search_json(
|
||||||
r'(?s)\b(?:playerC|c)onfig\s*=\s*({.+?})\s*[;\n]', webpage, 'info section'), video_id)
|
r'\b(?:playerC|c)onfig\s*=', webpage, 'info section', video_id)
|
||||||
if config.get('view') == 4:
|
if config.get('view') == 4:
|
||||||
config = self._verify_player_video_password(
|
config = self._verify_player_video_password(
|
||||||
redirect_url, video_id, headers)
|
redirect_url, video_id, headers)
|
||||||
|
|
|
@ -4,20 +4,28 @@ from __future__ import unicode_literals
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_chr
|
from ..compat import (
|
||||||
|
compat_chr,
|
||||||
|
compat_zip as zip,
|
||||||
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
clean_html,
|
||||||
decode_packed_codes,
|
decode_packed_codes,
|
||||||
determine_ext,
|
determine_ext,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
get_element_by_id,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
js_to_json,
|
merge_dicts,
|
||||||
|
T,
|
||||||
|
traverse_obj,
|
||||||
|
url_or_none,
|
||||||
urlencode_postdata,
|
urlencode_postdata,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# based on openload_decode from 2bfeee69b976fe049761dd3012e30b637ee05a58
|
# based on openload_decode from 2bfeee69b976fe049761dd3012e30b637ee05a58
|
||||||
def aa_decode(aa_code):
|
def aa_decode(aa_code):
|
||||||
symbol_table = [
|
symbol_table = (
|
||||||
('7', '((゚ー゚) + (o^_^o))'),
|
('7', '((゚ー゚) + (o^_^o))'),
|
||||||
('6', '((o^_^o) +(o^_^o))'),
|
('6', '((o^_^o) +(o^_^o))'),
|
||||||
('5', '((゚ー゚) + (゚Θ゚))'),
|
('5', '((゚ー゚) + (゚Θ゚))'),
|
||||||
|
@ -26,84 +34,180 @@ def aa_decode(aa_code):
|
||||||
('3', '(o^_^o)'),
|
('3', '(o^_^o)'),
|
||||||
('1', '(゚Θ゚)'),
|
('1', '(゚Θ゚)'),
|
||||||
('0', '(c^_^o)'),
|
('0', '(c^_^o)'),
|
||||||
]
|
('+', ''),
|
||||||
|
)
|
||||||
delim = '(゚Д゚)[゚ε゚]+'
|
delim = '(゚Д゚)[゚ε゚]+'
|
||||||
ret = ''
|
|
||||||
for aa_char in aa_code.split(delim):
|
def chr_from_code(c):
|
||||||
for val, pat in symbol_table:
|
for val, pat in symbol_table:
|
||||||
aa_char = aa_char.replace(pat, val)
|
c = c.replace(pat, val)
|
||||||
aa_char = aa_char.replace('+ ', '')
|
if c.startswith(('u', 'U')):
|
||||||
m = re.match(r'^\d+', aa_char)
|
base = 16
|
||||||
if m:
|
c = c[1:]
|
||||||
ret += compat_chr(int(m.group(0), 8))
|
|
||||||
else:
|
else:
|
||||||
m = re.match(r'^u([\da-f]+)', aa_char)
|
base = 10
|
||||||
if m:
|
c = int_or_none(c, base=base)
|
||||||
ret += compat_chr(int(m.group(1), 16))
|
return '' if c is None else compat_chr(c)
|
||||||
return ret
|
|
||||||
|
return ''.join(
|
||||||
|
chr_from_code(aa_char)
|
||||||
|
for aa_char in aa_code.split(delim))
|
||||||
|
|
||||||
|
|
||||||
class XFileShareIE(InfoExtractor):
|
class XFileShareIE(InfoExtractor):
|
||||||
_SITES = (
|
_SITES = (
|
||||||
(r'aparat\.cam', 'Aparat'),
|
# status check 2024-02: site availability, G site: search
|
||||||
(r'clipwatching\.com', 'ClipWatching'),
|
(r'aparat\.cam', 'Aparat'), # Cloudflare says host error 522, apparently changed to wolfstreeam.tv
|
||||||
(r'gounlimited\.to', 'GoUnlimited'),
|
(r'filemoon\.sx/.', 'FileMoon'),
|
||||||
(r'govid\.me', 'GoVid'),
|
(r'gounlimited\.to', 'GoUnlimited'), # no media pages listed
|
||||||
(r'holavid\.com', 'HolaVid'),
|
(r'govid\.me', 'GoVid'), # no media pages listed
|
||||||
(r'streamty\.com', 'Streamty'),
|
(r'highstream\.tv', 'HighStream'), # clipwatching.com redirects here
|
||||||
(r'thevideobee\.to', 'TheVideoBee'),
|
(r'holavid\.com', 'HolaVid'), # Cloudflare says host error 522
|
||||||
(r'uqload\.com', 'Uqload'),
|
# (r'streamty\.com', 'Streamty'), # no media pages listed, connection timeout
|
||||||
(r'vidbom\.com', 'VidBom'),
|
# (r'thevideobee\.to', 'TheVideoBee'), # no pages listed, refuses connection
|
||||||
(r'vidlo\.us', 'vidlo'),
|
(r'uqload\.to', 'Uqload'), # .com, .co redirect here
|
||||||
(r'vidlocker\.xyz', 'VidLocker'),
|
(r'(?:vedbam\.xyz|vadbam.net)', 'V?dB?m'), # vidbom.com redirects here, but no valid media pages listed
|
||||||
(r'vidshare\.tv', 'VidShare'),
|
(r'vidlo\.us', 'vidlo'), # no valid media pages listed
|
||||||
(r'vup\.to', 'VUp'),
|
(r'vidlocker\.xyz', 'VidLocker'), # no media pages listed
|
||||||
|
(r'(?:w\d\.)?viidshar\.com', 'VidShare'), # vidshare.tv redirects here
|
||||||
|
# (r'vup\.to', 'VUp'), # domain not found
|
||||||
(r'wolfstream\.tv', 'WolfStream'),
|
(r'wolfstream\.tv', 'WolfStream'),
|
||||||
(r'xvideosharing\.com', 'XVideoSharing'),
|
(r'xvideosharing\.com', 'XVideoSharing'), # just started showing 'maintenance mode'
|
||||||
)
|
)
|
||||||
|
|
||||||
IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
|
IE_DESC = 'XFileShare-based sites: %s' % ', '.join(list(zip(*_SITES))[1])
|
||||||
_VALID_URL = (r'https?://(?:www\.)?(?P<host>%s)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
|
_VALID_URL = (r'https?://(?:www\.)?(?P<host>%s)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
|
||||||
% '|'.join(site for site in list(zip(*_SITES))[0]))
|
% '|'.join(site for site in list(zip(*_SITES))[0]))
|
||||||
|
_EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:%s)/embed-[0-9a-zA-Z]+.*?)\1' % '|'.join(site for site in list(zip(*_SITES))[0])]
|
||||||
|
|
||||||
_FILE_NOT_FOUND_REGEXES = (
|
_FILE_NOT_FOUND_REGEXES = (
|
||||||
r'>(?:404 - )?File Not Found<',
|
r'>(?:404 - )?File Not Found<',
|
||||||
r'>The file was removed by administrator<',
|
r'>The file was removed by administrator<',
|
||||||
)
|
)
|
||||||
|
_TITLE_REGEXES = (
|
||||||
|
r'style="z-index: [0-9]+;">([^<]+)</span>',
|
||||||
|
r'<td nowrap>([^<]+)</td>',
|
||||||
|
r'h4-fine[^>]*>([^<]+)<',
|
||||||
|
r'>Watch (.+)[ <]',
|
||||||
|
r'<h2 class="video-page-head">([^<]+)</h2>',
|
||||||
|
r'<h2 style="[^"]*color:#403f3d[^"]*"[^>]*>([^<]+)<', # streamin.to (dead)
|
||||||
|
r'title\s*:\s*"([^"]+)"', # govid.me
|
||||||
|
)
|
||||||
|
_SOURCE_URL_REGEXES = (
|
||||||
|
r'(?:file|src)\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
|
||||||
|
r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
|
||||||
|
r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
|
||||||
|
r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
|
||||||
|
)
|
||||||
|
_THUMBNAIL_REGEXES = (
|
||||||
|
r'<video[^>]+poster="([^"]+)"',
|
||||||
|
r'(?:image|poster)\s*:\s*["\'](http[^"\']+)["\'],',
|
||||||
|
)
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://xvideosharing.com/fq65f94nd2ve',
|
'note': 'link in `sources`',
|
||||||
'md5': '4181f63957e8fe90ac836fa58dc3c8a6',
|
'url': 'https://uqload.to/dcsu06gdb45o',
|
||||||
|
'md5': '7f8db187b254379440bf4fcad094ae86',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'fq65f94nd2ve',
|
'id': 'dcsu06gdb45o',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'sample',
|
'title': 'f2e31015957e74c8c8427982e161c3fc mp4',
|
||||||
'thumbnail': r're:http://.*\.jpg',
|
'thumbnail': r're:https://.*\.jpg'
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'nocheckcertificate': True,
|
||||||
|
},
|
||||||
|
'expected_warnings': ['Unable to extract JWPlayer data'],
|
||||||
|
}, {
|
||||||
|
'note': 'link in decoded `sources`',
|
||||||
|
'url': 'https://xvideosharing.com/1tlg6agrrdgc',
|
||||||
|
'md5': '2608ce41932c1657ae56258a64e647d9',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '1tlg6agrrdgc',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': '0121',
|
||||||
|
'thumbnail': r're:https?://.*\.jpg',
|
||||||
|
},
|
||||||
|
'skip': 'This server is in maintenance mode.',
|
||||||
|
}, {
|
||||||
|
'note': 'JWPlayer link in un-p,a,c,k,e,d JS',
|
||||||
|
'url': 'https://filemoon.sx/e/dw40rxrzruqz',
|
||||||
|
'md5': '5a713742f57ac4aef29b74733e8dda01',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'dw40rxrzruqz',
|
||||||
|
'title': 'dw40rxrzruqz',
|
||||||
|
'ext': 'mp4'
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'note': 'JWPlayer link in un-p,a,c,k,e,d JS',
|
||||||
|
'url': 'https://vadbam.net/6lnbkci96wly.html',
|
||||||
|
'md5': 'a1616800076177e2ac769203957c54bc',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '6lnbkci96wly',
|
||||||
|
'title': 'Heart Crime S01 E03 weciima autos',
|
||||||
|
'ext': 'mp4'
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'note': 'JWPlayer link in clear',
|
||||||
|
'url': 'https://w1.viidshar.com/nnibe0xf0h79.html',
|
||||||
|
'md5': 'f0a580ce9df06cc61b4a5c979d672367',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'nnibe0xf0h79',
|
||||||
|
'title': 'JaGa 68ar',
|
||||||
|
'ext': 'mp4'
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'skip_download': 'ffmpeg',
|
||||||
|
},
|
||||||
|
'expected_warnings': ['hlsnative has detected features it does not support'],
|
||||||
|
}, {
|
||||||
|
'note': 'JWPlayer link in clear',
|
||||||
|
'url': 'https://wolfstream.tv/a3drtehyrg52.html',
|
||||||
|
'md5': '1901d86a79c5e0c6a51bdc9a4cfd3769',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'a3drtehyrg52',
|
||||||
|
'title': 'NFL 2023 W04 DET@GB',
|
||||||
|
'ext': 'mp4'
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://aparat.cam/n4d6dh0wvlpr',
|
'url': 'https://aparat.cam/n4d6dh0wvlpr',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://wolfstream.tv/nthme29v9u2x',
|
'url': 'https://uqload.to/ug5somm0ctnk.html',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://highstream.tv/2owiyz3sjoux',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://vedbam.xyz/6lnbkci96wly.html',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
@staticmethod
|
@classmethod
|
||||||
def _extract_urls(webpage):
|
def _extract_urls(cls, webpage):
|
||||||
return [
|
|
||||||
mobj.group('url')
|
def yield_urls():
|
||||||
for mobj in re.finditer(
|
for regex in cls._EMBED_REGEX:
|
||||||
r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:%s)/embed-[0-9a-zA-Z]+.*?)\1'
|
for mobj in re.finditer(regex, webpage):
|
||||||
% '|'.join(site for site in list(zip(*XFileShareIE._SITES))[0]),
|
yield mobj.group('url')
|
||||||
webpage)]
|
|
||||||
|
return list(yield_urls())
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
host, video_id = re.match(self._VALID_URL, url).groups()
|
host, video_id = self._match_valid_url(url).group('host', 'id')
|
||||||
|
|
||||||
url = 'https://%s/' % host + ('embed-%s.html' % video_id if host in ('govid.me', 'vidlo.us') else video_id)
|
url = 'https://%s/%s' % (
|
||||||
|
host,
|
||||||
|
'embed-%s.html' % video_id if host in ('govid.me', 'vidlo.us') else video_id)
|
||||||
webpage = self._download_webpage(url, video_id)
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
container_div = get_element_by_id('container', webpage) or webpage
|
||||||
if any(re.search(p, webpage) for p in self._FILE_NOT_FOUND_REGEXES):
|
if self._search_regex(
|
||||||
|
r'>This server is in maintenance mode\.', container_div,
|
||||||
|
'maint error', group=0, default=None):
|
||||||
|
raise ExtractorError(clean_html(container_div), expected=True)
|
||||||
|
if self._search_regex(
|
||||||
|
self._FILE_NOT_FOUND_REGEXES, container_div,
|
||||||
|
'missing video error', group=0, default=None):
|
||||||
raise ExtractorError('Video %s does not exist' % video_id, expected=True)
|
raise ExtractorError('Video %s does not exist' % video_id, expected=True)
|
||||||
|
|
||||||
fields = self._hidden_inputs(webpage)
|
fields = self._hidden_inputs(webpage)
|
||||||
|
@ -122,59 +226,43 @@ class XFileShareIE(InfoExtractor):
|
||||||
'Content-type': 'application/x-www-form-urlencoded',
|
'Content-type': 'application/x-www-form-urlencoded',
|
||||||
})
|
})
|
||||||
|
|
||||||
title = (self._search_regex(
|
title = (
|
||||||
(r'style="z-index: [0-9]+;">([^<]+)</span>',
|
self._search_regex(self._TITLE_REGEXES, webpage, 'title', default=None)
|
||||||
r'<td nowrap>([^<]+)</td>',
|
or self._og_search_title(webpage, default=None)
|
||||||
r'h4-fine[^>]*>([^<]+)<',
|
or video_id).strip()
|
||||||
r'>Watch (.+)[ <]',
|
|
||||||
r'<h2 class="video-page-head">([^<]+)</h2>',
|
|
||||||
r'<h2 style="[^"]*color:#403f3d[^"]*"[^>]*>([^<]+)<', # streamin.to
|
|
||||||
r'title\s*:\s*"([^"]+)"'), # govid.me
|
|
||||||
webpage, 'title', default=None) or self._og_search_title(
|
|
||||||
webpage, default=None) or video_id).strip()
|
|
||||||
|
|
||||||
for regex, func in (
|
obf_code = True
|
||||||
(r'(eval\(function\(p,a,c,k,e,d\){.+)', decode_packed_codes),
|
while obf_code:
|
||||||
(r'(゚.+)', aa_decode)):
|
for regex, func in (
|
||||||
obf_code = self._search_regex(regex, webpage, 'obfuscated code', default=None)
|
(r'(?s)(?<!-)\b(eval\(function\(p,a,c,k,e,d\)\{(?:(?!</script>).)+\)\))',
|
||||||
if obf_code:
|
decode_packed_codes),
|
||||||
webpage = webpage.replace(obf_code, func(obf_code))
|
(r'(゚.+)', aa_decode)):
|
||||||
|
obf_code = self._search_regex(regex, webpage, 'obfuscated code', default=None)
|
||||||
|
if obf_code:
|
||||||
|
webpage = webpage.replace(obf_code, func(obf_code))
|
||||||
|
break
|
||||||
|
|
||||||
formats = []
|
jwplayer_data = self._find_jwplayer_data(
|
||||||
|
webpage.replace(r'\'', '\''), video_id)
|
||||||
|
result = self._parse_jwplayer_data(
|
||||||
|
jwplayer_data, video_id, require_title=False,
|
||||||
|
m3u8_id='hls', mpd_id='dash')
|
||||||
|
|
||||||
jwplayer_data = self._search_regex(
|
if not traverse_obj(result, 'formats'):
|
||||||
[
|
|
||||||
r'jwplayer\("[^"]+"\)\.load\(\[({.+?})\]\);',
|
|
||||||
r'jwplayer\("[^"]+"\)\.setup\(({.+?})\);',
|
|
||||||
], webpage,
|
|
||||||
'jwplayer data', default=None)
|
|
||||||
if jwplayer_data:
|
|
||||||
jwplayer_data = self._parse_json(
|
|
||||||
jwplayer_data.replace(r"\'", "'"), video_id, js_to_json)
|
|
||||||
if jwplayer_data:
|
if jwplayer_data:
|
||||||
formats = self._parse_jwplayer_data(
|
self.report_warning(
|
||||||
jwplayer_data, video_id, False,
|
'Failed to extract JWPlayer formats', video_id=video_id)
|
||||||
m3u8_id='hls', mpd_id='dash')['formats']
|
urls = set()
|
||||||
|
for regex in self._SOURCE_URL_REGEXES:
|
||||||
if not formats:
|
|
||||||
urls = []
|
|
||||||
for regex in (
|
|
||||||
r'(?:file|src)\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
|
|
||||||
r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
|
|
||||||
r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
|
|
||||||
r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1'):
|
|
||||||
for mobj in re.finditer(regex, webpage):
|
for mobj in re.finditer(regex, webpage):
|
||||||
video_url = mobj.group('url')
|
urls.add(mobj.group('url'))
|
||||||
if video_url not in urls:
|
|
||||||
urls.append(video_url)
|
|
||||||
|
|
||||||
sources = self._search_regex(
|
sources = self._search_regex(
|
||||||
r'sources\s*:\s*(\[(?!{)[^\]]+\])', webpage, 'sources', default=None)
|
r'sources\s*:\s*(\[(?!{)[^\]]+\])', webpage, 'sources', default=None)
|
||||||
if sources:
|
urls.update(traverse_obj(sources, (T(lambda s: self._parse_json(s, video_id)), Ellipsis)))
|
||||||
urls.extend(self._parse_json(sources, video_id))
|
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for video_url in urls:
|
for video_url in traverse_obj(urls, (Ellipsis, T(url_or_none))):
|
||||||
if determine_ext(video_url) == 'm3u8':
|
if determine_ext(video_url) == 'm3u8':
|
||||||
formats.extend(self._extract_m3u8_formats(
|
formats.extend(self._extract_m3u8_formats(
|
||||||
video_url, video_id, 'mp4',
|
video_url, video_id, 'mp4',
|
||||||
|
@ -185,17 +273,19 @@ class XFileShareIE(InfoExtractor):
|
||||||
'url': video_url,
|
'url': video_url,
|
||||||
'format_id': 'sd',
|
'format_id': 'sd',
|
||||||
})
|
})
|
||||||
self._sort_formats(formats)
|
result = {'formats': formats}
|
||||||
|
|
||||||
|
self._sort_formats(result['formats'])
|
||||||
|
|
||||||
thumbnail = self._search_regex(
|
thumbnail = self._search_regex(
|
||||||
[
|
self._THUMBNAIL_REGEXES, webpage, 'thumbnail', default=None)
|
||||||
r'<video[^>]+poster="([^"]+)"',
|
|
||||||
r'(?:image|poster)\s*:\s*["\'](http[^"\']+)["\'],',
|
|
||||||
], webpage, 'thumbnail', default=None)
|
|
||||||
|
|
||||||
return {
|
if not (title or result.get('title')):
|
||||||
|
title = self._generic_title(url) or video_id
|
||||||
|
|
||||||
|
return merge_dicts(result, {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': title,
|
'title': title or None,
|
||||||
'thumbnail': thumbnail,
|
'thumbnail': thumbnail,
|
||||||
'formats': formats,
|
'http_headers': {'Referer': url}
|
||||||
}
|
})
|
||||||
|
|
|
@ -1,20 +1,38 @@
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import itertools
|
||||||
import re
|
import re
|
||||||
|
from time import sleep
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
clean_html,
|
||||||
extract_attributes,
|
extract_attributes,
|
||||||
|
ExtractorError,
|
||||||
|
get_element_by_class,
|
||||||
|
get_element_by_id,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
str_to_int,
|
merge_dicts,
|
||||||
|
parse_count,
|
||||||
|
parse_qs,
|
||||||
|
T,
|
||||||
|
traverse_obj,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
url_or_none,
|
url_or_none,
|
||||||
|
urljoin,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class YouPornIE(InfoExtractor):
|
class YouPornIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
|
_VALID_URL = (
|
||||||
|
r'youporn:(?P<id>\d+)',
|
||||||
|
r'''(?x)
|
||||||
|
https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)
|
||||||
|
(?:/(?:(?P<display_id>[^/?#&]+)/?)?)?(?:[#?]|$)
|
||||||
|
'''
|
||||||
|
)
|
||||||
|
_EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?youporn\.com/embed/\d+)']
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
|
'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
|
||||||
'md5': '3744d24c50438cf5b6f6d59feb5055c2',
|
'md5': '3744d24c50438cf5b6f6d59feb5055c2',
|
||||||
|
@ -34,7 +52,7 @@ class YouPornIE(InfoExtractor):
|
||||||
'tags': list,
|
'tags': list,
|
||||||
'age_limit': 18,
|
'age_limit': 18,
|
||||||
},
|
},
|
||||||
'skip': 'This video has been disabled',
|
'skip': 'This video has been deactivated',
|
||||||
}, {
|
}, {
|
||||||
# Unknown uploader
|
# Unknown uploader
|
||||||
'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
|
'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
|
||||||
|
@ -66,57 +84,104 @@ class YouPornIE(InfoExtractor):
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://www.youporn.com/watch/13922959/femdom-principal/',
|
'url': 'https://www.youporn.com/watch/13922959/femdom-principal/',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.youporn.com/watch/16290308/tinderspecial-trailer1/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '16290308',
|
||||||
|
'age_limit': 18,
|
||||||
|
'categories': [],
|
||||||
|
'description': None, # SEO spam using title removed
|
||||||
|
'display_id': 'tinderspecial-trailer1',
|
||||||
|
'duration': 298.0,
|
||||||
|
'ext': 'mp4',
|
||||||
|
'upload_date': '20201123',
|
||||||
|
'uploader': 'Ersties',
|
||||||
|
'tags': [],
|
||||||
|
'thumbnail': 'https://fi1.ypncdn.com/m=eaSaaTbWx/202011/23/16290308/original/3.jpg',
|
||||||
|
'timestamp': 1606147564,
|
||||||
|
'title': 'Tinder In Real Life',
|
||||||
|
'view_count': int,
|
||||||
|
}
|
||||||
}]
|
}]
|
||||||
|
|
||||||
@staticmethod
|
@classmethod
|
||||||
def _extract_urls(webpage):
|
def _extract_urls(cls, webpage):
|
||||||
return re.findall(
|
def yield_urls():
|
||||||
r'<iframe[^>]+\bsrc=["\']((?:https?:)?//(?:www\.)?youporn\.com/embed/\d+)',
|
for p in cls._EMBED_REGEX:
|
||||||
webpage)
|
for m in re.finditer(p, webpage):
|
||||||
|
yield m.group('url')
|
||||||
|
|
||||||
|
return list(yield_urls())
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
# A different video ID (data-video-id) is hidden in the page but
|
||||||
video_id = mobj.group('id')
|
# never seems to be used
|
||||||
display_id = mobj.group('display_id') or video_id
|
video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
|
||||||
|
url = 'http://www.youporn.com/watch/%s' % (video_id,)
|
||||||
|
webpage = self._download_webpage(
|
||||||
|
url, video_id, headers={'Cookie': 'age_verified=1'})
|
||||||
|
|
||||||
definitions = self._download_json(
|
watchable = self._search_regex(
|
||||||
'https://www.youporn.com/api/video/media_definitions/%s/' % video_id,
|
r'''(<div\s[^>]*\bid\s*=\s*('|")?watch-container(?(2)\2|(?!-)\b)[^>]*>)''',
|
||||||
display_id)
|
webpage, 'watchability', default=None)
|
||||||
|
if not watchable:
|
||||||
|
msg = re.split(r'\s{4}', clean_html(get_element_by_id(
|
||||||
|
'mainContent', webpage)) or '')[0]
|
||||||
|
raise ExtractorError(
|
||||||
|
('%s says: %s' % (self.IE_NAME, msg))
|
||||||
|
if msg else 'Video unavailable: no reason found',
|
||||||
|
expected=True)
|
||||||
|
# internal ID ?
|
||||||
|
# video_id = extract_attributes(watchable).get('data-video-id')
|
||||||
|
|
||||||
|
playervars = self._search_json(
|
||||||
|
r'\bplayervars\s*:', webpage, 'playervars', video_id)
|
||||||
|
|
||||||
|
def get_fmt(x):
|
||||||
|
v_url = url_or_none(x.get('videoUrl'))
|
||||||
|
if v_url:
|
||||||
|
x['videoUrl'] = v_url
|
||||||
|
return (x['format'], x)
|
||||||
|
|
||||||
|
defs_by_format = dict(traverse_obj(playervars, (
|
||||||
|
'mediaDefinitions', lambda _, v: v.get('format'), T(get_fmt))))
|
||||||
|
|
||||||
|
def get_format_data(f):
|
||||||
|
if f not in defs_by_format:
|
||||||
|
return []
|
||||||
|
return self._download_json(
|
||||||
|
defs_by_format[f]['videoUrl'], video_id, '{0}-formats'.format(f))
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for definition in definitions:
|
# Try to extract only the actual master m3u8 first, avoiding the duplicate single resolution "master" m3u8s
|
||||||
if not isinstance(definition, dict):
|
for hls_url in traverse_obj(
|
||||||
continue
|
get_format_data('hls'),
|
||||||
video_url = url_or_none(definition.get('videoUrl'))
|
(lambda _, v: not isinstance(v['defaultQuality'], bool), 'videoUrl'),
|
||||||
if not video_url:
|
(Ellipsis, 'videoUrl')):
|
||||||
continue
|
formats.extend(self._extract_m3u8_formats(
|
||||||
f = {
|
hls_url, video_id, 'mp4', fatal=False, m3u8_id='hls',
|
||||||
'url': video_url,
|
entry_protocol='m3u8_native'))
|
||||||
'filesize': int_or_none(definition.get('videoSize')),
|
|
||||||
}
|
for f in traverse_obj(get_format_data('mp4'), (
|
||||||
height = int_or_none(definition.get('quality'))
|
lambda _, v: v.get('videoUrl'), {
|
||||||
|
'url': ('videoUrl', T(url_or_none)),
|
||||||
|
'filesize': ('videoSize', T(int_or_none)),
|
||||||
|
'height': ('quality', T(int_or_none)),
|
||||||
|
}, T(lambda x: x.get('videoUrl') and x))):
|
||||||
# Video URL's path looks like this:
|
# Video URL's path looks like this:
|
||||||
# /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
|
# /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
|
||||||
# /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
|
# /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
|
||||||
# /videos/201703/11/109285532/1080P_4000K_109285532.mp4
|
# /videos/201703/11/109285532/1080P_4000K_109285532.mp4
|
||||||
# We will benefit from it by extracting some metadata
|
# We will benefit from it by extracting some metadata
|
||||||
mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
|
mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', f['videoUrl'])
|
||||||
if mobj:
|
if mobj:
|
||||||
if not height:
|
if not f.get('height'):
|
||||||
height = int(mobj.group('height'))
|
f['height'] = int(mobj.group('height'))
|
||||||
bitrate = int(mobj.group('bitrate'))
|
f['tbr'] = int(mobj.group('bitrate'))
|
||||||
f.update({
|
f['format_id'] = '%dp-%dk' % (f['height'], f['tbr'])
|
||||||
'format_id': '%dp-%dk' % (height, bitrate),
|
|
||||||
'tbr': bitrate,
|
|
||||||
})
|
|
||||||
f['height'] = height
|
|
||||||
formats.append(f)
|
formats.append(f)
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
webpage = self._download_webpage(
|
|
||||||
'http://www.youporn.com/watch/%s' % video_id, display_id,
|
|
||||||
headers={'Cookie': 'age_verified=1'})
|
|
||||||
|
|
||||||
title = self._html_search_regex(
|
title = self._html_search_regex(
|
||||||
r'(?s)<div[^>]+class=["\']watchVideoTitle[^>]+>(.+?)</div>',
|
r'(?s)<div[^>]+class=["\']watchVideoTitle[^>]+>(.+?)</div>',
|
||||||
webpage, 'title', default=None) or self._og_search_title(
|
webpage, 'title', default=None) or self._og_search_title(
|
||||||
|
@ -131,8 +196,10 @@ class YouPornIE(InfoExtractor):
|
||||||
thumbnail = self._search_regex(
|
thumbnail = self._search_regex(
|
||||||
r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
|
r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
|
||||||
webpage, 'thumbnail', fatal=False, group='thumbnail')
|
webpage, 'thumbnail', fatal=False, group='thumbnail')
|
||||||
duration = int_or_none(self._html_search_meta(
|
duration = traverse_obj(playervars, ('duration', T(int_or_none)))
|
||||||
'video:duration', webpage, 'duration', fatal=False))
|
if duration is None:
|
||||||
|
duration = int_or_none(self._html_search_meta(
|
||||||
|
'video:duration', webpage, 'duration', fatal=False))
|
||||||
|
|
||||||
uploader = self._html_search_regex(
|
uploader = self._html_search_regex(
|
||||||
r'(?s)<div[^>]+class=["\']submitByLink["\'][^>]*>(.+?)</div>',
|
r'(?s)<div[^>]+class=["\']submitByLink["\'][^>]*>(.+?)</div>',
|
||||||
|
@ -148,11 +215,11 @@ class YouPornIE(InfoExtractor):
|
||||||
|
|
||||||
view_count = None
|
view_count = None
|
||||||
views = self._search_regex(
|
views = self._search_regex(
|
||||||
r'(<div[^>]+\bclass=["\']js_videoInfoViews["\']>)', webpage,
|
r'(<div\s[^>]*\bdata-value\s*=[^>]+>)\s*<label>Views:</label>',
|
||||||
'views', default=None)
|
webpage, 'views', default=None)
|
||||||
if views:
|
if views:
|
||||||
view_count = str_to_int(extract_attributes(views).get('data-value'))
|
view_count = parse_count(extract_attributes(views).get('data-value'))
|
||||||
comment_count = str_to_int(self._search_regex(
|
comment_count = parse_count(self._search_regex(
|
||||||
r'>All [Cc]omments? \(([\d,.]+)\)',
|
r'>All [Cc]omments? \(([\d,.]+)\)',
|
||||||
webpage, 'comment count', default=None))
|
webpage, 'comment count', default=None))
|
||||||
|
|
||||||
|
@ -168,7 +235,10 @@ class YouPornIE(InfoExtractor):
|
||||||
r'(?s)Tags:.*?</div>\s*<div[^>]+class=["\']tagBoxContent["\'][^>]*>(.+?)</div>',
|
r'(?s)Tags:.*?</div>\s*<div[^>]+class=["\']tagBoxContent["\'][^>]*>(.+?)</div>',
|
||||||
'tags')
|
'tags')
|
||||||
|
|
||||||
return {
|
data = self._search_json_ld(webpage, video_id, expected_type='VideoObject', fatal=False) or {}
|
||||||
|
data.pop('url', None)
|
||||||
|
|
||||||
|
result = merge_dicts(data, {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'display_id': display_id,
|
'display_id': display_id,
|
||||||
'title': title,
|
'title': title,
|
||||||
|
@ -183,4 +253,442 @@ class YouPornIE(InfoExtractor):
|
||||||
'tags': tags,
|
'tags': tags,
|
||||||
'age_limit': age_limit,
|
'age_limit': age_limit,
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
}
|
})
|
||||||
|
# Remove promotional non-description
|
||||||
|
if result.get('description', '').startswith(
|
||||||
|
'Watch %s online' % (result['title'],)):
|
||||||
|
del result['description']
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class YouPornListBase(InfoExtractor):
|
||||||
|
# pattern in '.title-text' element of page section containing videos
|
||||||
|
_PLAYLIST_TITLEBAR_RE = r'\s+[Vv]ideos\s*$'
|
||||||
|
_PAGE_RETRY_COUNT = 0 # ie, no retry
|
||||||
|
_PAGE_RETRY_DELAY = 2 # seconds
|
||||||
|
|
||||||
|
def _get_next_url(self, url, pl_id, html):
|
||||||
|
return urljoin(url, self._search_regex(
|
||||||
|
r'''<a\s[^>]*?\bhref\s*=\s*("|')(?P<url>(?:(?!\1)[^>])+)\1''',
|
||||||
|
get_element_by_id('next', html) or '', 'next page',
|
||||||
|
group='url', default=None))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_title_from_slug(cls, title_slug):
|
||||||
|
return re.sub(r'[_-]', ' ', title_slug)
|
||||||
|
|
||||||
|
def _entries(self, url, pl_id, html=None, page_num=None):
|
||||||
|
|
||||||
|
# separates page sections
|
||||||
|
PLAYLIST_SECTION_RE = (
|
||||||
|
r'''<div\s[^>]*\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?title-bar(?:\s+[\w$-]+|\s)*\1[^>]*>'''
|
||||||
|
)
|
||||||
|
# contains video link
|
||||||
|
VIDEO_URL_RE = r'''(?x)
|
||||||
|
<div\s[^>]*\bdata-video-id\s*=\s*('|")\d+\1[^>]*>\s*
|
||||||
|
(?:<div\b[\s\S]+?</div>\s*)*
|
||||||
|
<a\s[^>]*\bhref\s*=\s*('|")(?P<url>(?:(?!\2)[^>])+)\2
|
||||||
|
'''
|
||||||
|
|
||||||
|
def yield_pages(url, html=html, page_num=page_num):
|
||||||
|
fatal = not html
|
||||||
|
for pnum in itertools.count(start=page_num or 1):
|
||||||
|
if not html:
|
||||||
|
html = self._download_webpage(
|
||||||
|
url, pl_id, note='Downloading page %d' % pnum,
|
||||||
|
fatal=fatal)
|
||||||
|
if not html:
|
||||||
|
break
|
||||||
|
fatal = False
|
||||||
|
yield (url, html, pnum)
|
||||||
|
# explicit page: extract just that page
|
||||||
|
if page_num is not None:
|
||||||
|
break
|
||||||
|
next_url = self._get_next_url(url, pl_id, html)
|
||||||
|
if not next_url or next_url == url:
|
||||||
|
break
|
||||||
|
url, html = next_url, None
|
||||||
|
|
||||||
|
def retry_page(msg, tries_left, page_data):
|
||||||
|
if tries_left <= 0:
|
||||||
|
return
|
||||||
|
self.report_warning(msg, pl_id)
|
||||||
|
sleep(self._PAGE_RETRY_DELAY)
|
||||||
|
return next(
|
||||||
|
yield_pages(page_data[0], page_num=page_data[2]), None)
|
||||||
|
|
||||||
|
def yield_entries(html):
|
||||||
|
for frag in re.split(PLAYLIST_SECTION_RE, html):
|
||||||
|
if not frag:
|
||||||
|
continue
|
||||||
|
t_text = get_element_by_class('title-text', frag or '')
|
||||||
|
if not (t_text and re.search(self._PLAYLIST_TITLEBAR_RE, t_text)):
|
||||||
|
continue
|
||||||
|
for m in re.finditer(VIDEO_URL_RE, frag):
|
||||||
|
video_url = urljoin(url, m.group('url'))
|
||||||
|
if video_url:
|
||||||
|
yield self.url_result(video_url)
|
||||||
|
|
||||||
|
last_first_url = None
|
||||||
|
for page_data in yield_pages(url, html=html, page_num=page_num):
|
||||||
|
# page_data: url, html, page_num
|
||||||
|
first_url = None
|
||||||
|
tries_left = self._PAGE_RETRY_COUNT + 1
|
||||||
|
while tries_left > 0:
|
||||||
|
tries_left -= 1
|
||||||
|
for from_ in yield_entries(page_data[1]):
|
||||||
|
# may get the same page twice instead of empty page
|
||||||
|
# or (site bug) intead of actual next page
|
||||||
|
if not first_url:
|
||||||
|
first_url = from_['url']
|
||||||
|
if first_url == last_first_url:
|
||||||
|
# sometimes (/porntags/) the site serves the previous page
|
||||||
|
# instead but may provide the correct page after a delay
|
||||||
|
page_data = retry_page(
|
||||||
|
'Retrying duplicate page...', tries_left, page_data)
|
||||||
|
if page_data:
|
||||||
|
first_url = None
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
yield from_
|
||||||
|
else:
|
||||||
|
if not first_url and 'no-result-paragarph1' in page_data[1]:
|
||||||
|
page_data = retry_page(
|
||||||
|
'Retrying empty page...', tries_left, page_data)
|
||||||
|
if page_data:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
# success/failure
|
||||||
|
break
|
||||||
|
# may get an infinite (?) sequence of empty pages
|
||||||
|
if not first_url:
|
||||||
|
break
|
||||||
|
last_first_url = first_url
|
||||||
|
|
||||||
|
def _real_extract(self, url, html=None):
|
||||||
|
# exceptionally, id may be None
|
||||||
|
m_dict = self._match_valid_url(url).groupdict()
|
||||||
|
pl_id, page_type, sort = (m_dict.get(k) for k in ('id', 'type', 'sort'))
|
||||||
|
|
||||||
|
qs = parse_qs(url)
|
||||||
|
for q, v in qs.items():
|
||||||
|
if v:
|
||||||
|
qs[q] = v[-1]
|
||||||
|
else:
|
||||||
|
del qs[q]
|
||||||
|
|
||||||
|
base_id = pl_id or 'YouPorn'
|
||||||
|
title = self._get_title_from_slug(base_id)
|
||||||
|
if page_type:
|
||||||
|
title = '%s %s' % (page_type.capitalize(), title)
|
||||||
|
base_id = [base_id.lower()]
|
||||||
|
if sort is None:
|
||||||
|
title += ' videos'
|
||||||
|
else:
|
||||||
|
title = '%s videos by %s' % (title, re.sub(r'[_-]', ' ', sort))
|
||||||
|
base_id.append(sort)
|
||||||
|
if qs:
|
||||||
|
ps = ['%s=%s' % item for item in sorted(qs.items())]
|
||||||
|
title += ' (%s)' % ','.join(ps)
|
||||||
|
base_id.extend(ps)
|
||||||
|
pl_id = '/'.join(base_id)
|
||||||
|
|
||||||
|
return self.playlist_result(
|
||||||
|
self._entries(url, pl_id, html=html,
|
||||||
|
page_num=int_or_none(qs.get('page'))),
|
||||||
|
playlist_id=pl_id, playlist_title=title)
|
||||||
|
|
||||||
|
|
||||||
|
class YouPornCategoryIE(YouPornListBase):
|
||||||
|
IE_DESC = 'YouPorn category, with sorting, filtering and pagination'
|
||||||
|
_VALID_URL = r'''(?x)
|
||||||
|
https?://(?:www\.)?youporn\.com/
|
||||||
|
(?P<type>category)/(?P<id>[^/?#&]+)
|
||||||
|
(?:/(?P<sort>popular|views|rating|time|duration))?/?(?:[#?]|$)
|
||||||
|
'''
|
||||||
|
_TESTS = [{
|
||||||
|
'note': 'Full list with pagination',
|
||||||
|
'url': 'https://www.youporn.com/category/lingerie/popular/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'lingerie/popular',
|
||||||
|
'title': 'Category lingerie videos by popular',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 39,
|
||||||
|
}, {
|
||||||
|
'note': 'Filtered paginated list with single page result',
|
||||||
|
'url': 'https://www.youporn.com/category/lingerie/duration/?min_minutes=10',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'lingerie/duration/min_minutes=10',
|
||||||
|
'title': 'Category lingerie videos by duration (min_minutes=10)',
|
||||||
|
},
|
||||||
|
'playlist_maxcount': 30,
|
||||||
|
}, {
|
||||||
|
'note': 'Single page of full list',
|
||||||
|
'url': 'https://www.youporn.com/category/lingerie/popular?page=1',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'lingerie/popular/page=1',
|
||||||
|
'title': 'Category lingerie videos by popular (page=1)',
|
||||||
|
},
|
||||||
|
'playlist_count': 30,
|
||||||
|
}]
|
||||||
|
|
||||||
|
|
||||||
|
class YouPornChannelIE(YouPornListBase):
|
||||||
|
IE_DESC = 'YouPorn channel, with sorting and pagination'
|
||||||
|
_VALID_URL = r'''(?x)
|
||||||
|
https?://(?:www\.)?youporn\.com/
|
||||||
|
(?P<type>channel)/(?P<id>[^/?#&]+)
|
||||||
|
(?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
|
||||||
|
'''
|
||||||
|
_TESTS = [{
|
||||||
|
'note': 'Full list with pagination',
|
||||||
|
'url': 'https://www.youporn.com/channel/x-feeds/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'x-feeds',
|
||||||
|
'title': 'Channel X-Feeds videos',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 37,
|
||||||
|
}, {
|
||||||
|
'note': 'Single page of full list (no filters here)',
|
||||||
|
'url': 'https://www.youporn.com/channel/x-feeds/duration?page=1',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'x-feeds/duration/page=1',
|
||||||
|
'title': 'Channel X-Feeds videos by duration (page=1)',
|
||||||
|
},
|
||||||
|
'playlist_count': 24,
|
||||||
|
}]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_title_from_slug(title_slug):
|
||||||
|
return re.sub(r'_', ' ', title_slug).title()
|
||||||
|
|
||||||
|
|
||||||
|
class YouPornCollectionIE(YouPornListBase):
|
||||||
|
IE_DESC = 'YouPorn collection (user playlist), with sorting and pagination'
|
||||||
|
_VALID_URL = r'''(?x)
|
||||||
|
https?://(?:www\.)?youporn\.com/
|
||||||
|
(?P<type>collection)s/videos/(?P<id>\d+)
|
||||||
|
(?:/(?P<sort>rating|views|time|duration))?/?(?:[#?]|$)
|
||||||
|
'''
|
||||||
|
_PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+in\s'
|
||||||
|
_TESTS = [{
|
||||||
|
'note': 'Full list with pagination',
|
||||||
|
'url': 'https://www.youporn.com/collections/videos/33044251/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '33044251',
|
||||||
|
'title': 'Collection Sexy Lips videos',
|
||||||
|
'uploader': 'ph-littlewillyb',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 50,
|
||||||
|
}, {
|
||||||
|
'note': 'Single page of full list (no filters here)',
|
||||||
|
'url': 'https://www.youporn.com/collections/videos/33044251/time?page=1',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '33044251/time/page=1',
|
||||||
|
'title': 'Collection Sexy Lips videos by time (page=1)',
|
||||||
|
'uploader': 'ph-littlewillyb',
|
||||||
|
},
|
||||||
|
'playlist_count': 20,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
pl_id = self._match_id(url)
|
||||||
|
html = self._download_webpage(url, pl_id)
|
||||||
|
playlist = super(YouPornCollectionIE, self)._real_extract(url, html=html)
|
||||||
|
infos = re.sub(r'\s+', ' ', clean_html(get_element_by_class(
|
||||||
|
'collection-infos', html)) or '')
|
||||||
|
title, uploader = self._search_regex(
|
||||||
|
r'^\s*Collection: (?P<title>.+?) \d+ VIDEOS \d+ VIEWS \d+ days LAST UPDATED From: (?P<uploader>[\w_-]+)',
|
||||||
|
infos, 'title/uploader', group=('title', 'uploader'), default=(None, None))
|
||||||
|
|
||||||
|
return merge_dicts({
|
||||||
|
'title': playlist['title'].replace(playlist['id'].split('/')[0], title),
|
||||||
|
'uploader': uploader,
|
||||||
|
}, playlist) if title else playlist
|
||||||
|
|
||||||
|
|
||||||
|
class YouPornTagIE(YouPornListBase):
|
||||||
|
IE_DESC = 'YouPorn tag (porntags), with sorting, filtering and pagination'
|
||||||
|
_VALID_URL = r'''(?x)
|
||||||
|
https?://(?:www\.)?youporn\.com/
|
||||||
|
porn(?P<type>tag)s/(?P<id>[^/?#&]+)
|
||||||
|
(?:/(?P<sort>views|rating|time|duration))?/?(?:[#?]|$)
|
||||||
|
'''
|
||||||
|
_PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+tagged\s'
|
||||||
|
_PAGE_RETRY_COUNT = 1
|
||||||
|
_TESTS = [{
|
||||||
|
'note': 'Full list with pagination',
|
||||||
|
'url': 'https://www.youporn.com/porntags/austrian',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'austrian',
|
||||||
|
'title': 'Tag austrian videos',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 35,
|
||||||
|
'expected_warnings': ['Retrying duplicate page'],
|
||||||
|
}, {
|
||||||
|
'note': 'Filtered paginated list with single page result',
|
||||||
|
'url': 'https://www.youporn.com/porntags/austrian/duration/?min_minutes=10',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'austrian/duration/min_minutes=10',
|
||||||
|
'title': 'Tag austrian videos by duration (min_minutes=10)',
|
||||||
|
},
|
||||||
|
# number of videos per page is (row x col) 2x3 + 6x4 + 2, or + 3,
|
||||||
|
# or more, varying with number of ads; let's set max as 9x4
|
||||||
|
# NB col 1 may not be shown in non-JS page with site CSS and zoom 100%
|
||||||
|
'playlist_maxcount': 32,
|
||||||
|
'expected_warnings': ['Retrying duplicate page', 'Retrying empty page'],
|
||||||
|
}, {
|
||||||
|
'note': 'Single page of full list',
|
||||||
|
'url': 'https://www.youporn.com/porntags/austrian/?page=1',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'austrian/page=1',
|
||||||
|
'title': 'Tag austrian videos (page=1)',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 32,
|
||||||
|
'playlist_maxcount': 34,
|
||||||
|
'expected_warnings': ['Retrying duplicate page', 'Retrying empty page'],
|
||||||
|
}]
|
||||||
|
|
||||||
|
# YP tag navigation is broken, loses sort
|
||||||
|
def _get_next_url(self, url, pl_id, html):
|
||||||
|
next_url = super(YouPornTagIE, self)._get_next_url(url, pl_id, html)
|
||||||
|
if next_url:
|
||||||
|
n = self._match_valid_url(next_url)
|
||||||
|
if n:
|
||||||
|
s = n.groupdict().get('sort')
|
||||||
|
if s:
|
||||||
|
u = self._match_valid_url(url)
|
||||||
|
if u:
|
||||||
|
u = u.groupdict().get('sort')
|
||||||
|
if s and not u:
|
||||||
|
n = n.end('sort')
|
||||||
|
next_url = next_url[:n] + '/' + u + next_url[n:]
|
||||||
|
return next_url
|
||||||
|
|
||||||
|
|
||||||
|
class YouPornStarIE(YouPornListBase):
|
||||||
|
IE_DESC = 'YouPorn Pornstar, with description, sorting and pagination'
|
||||||
|
_VALID_URL = r'''(?x)
|
||||||
|
https?://(?:www\.)?youporn\.com/
|
||||||
|
(?P<type>pornstar)/(?P<id>[^/?#&]+)
|
||||||
|
(?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
|
||||||
|
'''
|
||||||
|
_PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+[fF]eaturing\s'
|
||||||
|
_TESTS = [{
|
||||||
|
'note': 'Full list with pagination',
|
||||||
|
'url': 'https://www.youporn.com/pornstar/daynia/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'daynia',
|
||||||
|
'title': 'Pornstar Daynia videos',
|
||||||
|
'description': r're:Daynia Rank \d+ Videos \d+ Views [\d,.]+ .+ Subscribers \d+',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 45,
|
||||||
|
}, {
|
||||||
|
'note': 'Single page of full list (no filters here)',
|
||||||
|
'url': 'https://www.youporn.com/pornstar/daynia/?page=1',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'daynia/page=1',
|
||||||
|
'title': 'Pornstar Daynia videos (page=1)',
|
||||||
|
'description': 're:.{180,}',
|
||||||
|
},
|
||||||
|
'playlist_count': 26,
|
||||||
|
}]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_title_from_slug(title_slug):
|
||||||
|
return re.sub(r'_', ' ', title_slug).title()
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
pl_id = self._match_id(url)
|
||||||
|
html = self._download_webpage(url, pl_id)
|
||||||
|
playlist = super(YouPornStarIE, self)._real_extract(url, html=html)
|
||||||
|
INFO_ELEMENT_RE = r'''(?x)
|
||||||
|
<div\s[^>]*\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?pornstar-info-wrapper(?:\s+[\w$-]+|\s)*\1[^>]*>
|
||||||
|
(?P<info>[\s\S]+?)(?:</div>\s*){6,}
|
||||||
|
'''
|
||||||
|
|
||||||
|
infos = self._search_regex(INFO_ELEMENT_RE, html, 'infos', group='info', default='')
|
||||||
|
if infos:
|
||||||
|
infos = re.sub(
|
||||||
|
r'(?:\s*nl=nl)+\s*', ' ',
|
||||||
|
re.sub(r'(?u)\s+', ' ', clean_html(
|
||||||
|
re.sub('\n', 'nl=nl', infos)))).replace('ribe Subsc', '')
|
||||||
|
|
||||||
|
return merge_dicts({
|
||||||
|
'description': infos.strip() or None,
|
||||||
|
}, playlist)
|
||||||
|
|
||||||
|
|
||||||
|
class YouPornVideosIE(YouPornListBase):
|
||||||
|
IE_DESC = 'YouPorn video (browse) playlists, with sorting, filtering and pagination'
|
||||||
|
_VALID_URL = r'''(?x)
|
||||||
|
https?://(?:www\.)?youporn\.com/
|
||||||
|
(?:(?P<id>browse)/)?
|
||||||
|
(?P<sort>(?(id)
|
||||||
|
(?:duration|rating|time|views)|
|
||||||
|
(?:most_(?:favou?rit|view)ed|recommended|top_rated)?))
|
||||||
|
(?:[/#?]|$)
|
||||||
|
'''
|
||||||
|
_PLAYLIST_TITLEBAR_RE = r'\s+(?:[Vv]ideos|VIDEOS)\s*$'
|
||||||
|
_TESTS = [{
|
||||||
|
'note': 'Full list with pagination (too long for test)',
|
||||||
|
'url': 'https://www.youporn.com/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'youporn',
|
||||||
|
'title': 'YouPorn videos',
|
||||||
|
},
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'note': 'Full list with pagination (too long for test)',
|
||||||
|
'url': 'https://www.youporn.com/recommended',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'youporn/recommended',
|
||||||
|
'title': 'YouPorn videos by recommended',
|
||||||
|
},
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'note': 'Full list with pagination (too long for test)',
|
||||||
|
'url': 'https://www.youporn.com/top_rated',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'youporn/top_rated',
|
||||||
|
'title': 'YouPorn videos by top rated',
|
||||||
|
},
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'note': 'Full list with pagination (too long for test)',
|
||||||
|
'url': 'https://www.youporn.com/browse/time',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'browse/time',
|
||||||
|
'title': 'YouPorn videos by time',
|
||||||
|
},
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'note': 'Filtered paginated list with single page result',
|
||||||
|
'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=2',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'youporn/most_favorited/max_minutes=2/res=VR',
|
||||||
|
'title': 'YouPorn videos by most favorited (max_minutes=2,res=VR)',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 10,
|
||||||
|
'playlist_maxcount': 28,
|
||||||
|
}, {
|
||||||
|
'note': 'Filtered paginated list with several pages',
|
||||||
|
'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=5',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'youporn/most_favorited/max_minutes=5/res=VR',
|
||||||
|
'title': 'YouPorn videos by most favorited (max_minutes=5,res=VR)',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 45,
|
||||||
|
}, {
|
||||||
|
'note': 'Single page of full list',
|
||||||
|
'url': 'https://www.youporn.com/browse/time?page=1',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'browse/time/page=1',
|
||||||
|
'title': 'YouPorn videos by time (page=1)',
|
||||||
|
},
|
||||||
|
'playlist_count': 36,
|
||||||
|
}]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_title_from_slug(title_slug):
|
||||||
|
return 'YouPorn' if title_slug == 'browse' else title_slug
|
||||||
|
|
|
@ -1647,10 +1647,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
except JSInterpreter.Exception as e:
|
except JSInterpreter.Exception as e:
|
||||||
self.report_warning(
|
self.report_warning(
|
||||||
'%s (%s %s)' % (
|
'%s (%s %s)' % (
|
||||||
self.__ie_msg(
|
'Unable to decode n-parameter: download likely to be throttled',
|
||||||
'Unable to decode n-parameter: download likely to be throttled'),
|
|
||||||
error_to_compat_str(e),
|
error_to_compat_str(e),
|
||||||
traceback.format_exc()))
|
traceback.format_exc()),
|
||||||
|
video_id=video_id)
|
||||||
return
|
return
|
||||||
|
|
||||||
self.write_debug('Decrypted nsig {0} => {1}'.format(n, ret))
|
self.write_debug('Decrypted nsig {0} => {1}'.format(n, ret))
|
||||||
|
|
|
@ -74,8 +74,11 @@ class FFmpegPostProcessor(PostProcessor):
|
||||||
return FFmpegPostProcessor(downloader)._versions
|
return FFmpegPostProcessor(downloader)._versions
|
||||||
|
|
||||||
def _determine_executables(self):
|
def _determine_executables(self):
|
||||||
programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
|
# ordered to match prefer_ffmpeg!
|
||||||
|
convs = ['ffmpeg', 'avconv']
|
||||||
|
probes = ['ffprobe', 'avprobe']
|
||||||
prefer_ffmpeg = True
|
prefer_ffmpeg = True
|
||||||
|
programs = convs + probes
|
||||||
|
|
||||||
def get_ffmpeg_version(path):
|
def get_ffmpeg_version(path):
|
||||||
ver = get_exe_version(path, args=['-version'])
|
ver = get_exe_version(path, args=['-version'])
|
||||||
|
@ -96,6 +99,7 @@ class FFmpegPostProcessor(PostProcessor):
|
||||||
|
|
||||||
self._paths = None
|
self._paths = None
|
||||||
self._versions = None
|
self._versions = None
|
||||||
|
location = None
|
||||||
if self._downloader:
|
if self._downloader:
|
||||||
prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', True)
|
prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', True)
|
||||||
location = self._downloader.params.get('ffmpeg_location')
|
location = self._downloader.params.get('ffmpeg_location')
|
||||||
|
@ -118,33 +122,21 @@ class FFmpegPostProcessor(PostProcessor):
|
||||||
location = os.path.dirname(os.path.abspath(location))
|
location = os.path.dirname(os.path.abspath(location))
|
||||||
if basename in ('ffmpeg', 'ffprobe'):
|
if basename in ('ffmpeg', 'ffprobe'):
|
||||||
prefer_ffmpeg = True
|
prefer_ffmpeg = True
|
||||||
|
self._paths = dict(
|
||||||
|
(p, p if location is None else os.path.join(location, p))
|
||||||
|
for p in programs)
|
||||||
|
self._versions = dict(
|
||||||
|
x for x in (
|
||||||
|
(p, get_ffmpeg_version(self._paths[p])) for p in programs)
|
||||||
|
if x[1] is not None)
|
||||||
|
|
||||||
self._paths = dict(
|
basenames = [None, None]
|
||||||
(p, os.path.join(location, p)) for p in programs)
|
for i, progs in enumerate((convs, probes)):
|
||||||
self._versions = dict(
|
for p in progs[::-1 if prefer_ffmpeg is False else 1]:
|
||||||
(p, get_ffmpeg_version(self._paths[p])) for p in programs)
|
if self._versions.get(p):
|
||||||
if self._versions is None:
|
basenames[i] = p
|
||||||
self._versions = dict(
|
break
|
||||||
(p, get_ffmpeg_version(p)) for p in programs)
|
self.basename, self.probe_basename = basenames
|
||||||
self._paths = dict((p, p) for p in programs)
|
|
||||||
|
|
||||||
if prefer_ffmpeg is False:
|
|
||||||
prefs = ('avconv', 'ffmpeg')
|
|
||||||
else:
|
|
||||||
prefs = ('ffmpeg', 'avconv')
|
|
||||||
for p in prefs:
|
|
||||||
if self._versions[p]:
|
|
||||||
self.basename = p
|
|
||||||
break
|
|
||||||
|
|
||||||
if prefer_ffmpeg is False:
|
|
||||||
prefs = ('avprobe', 'ffprobe')
|
|
||||||
else:
|
|
||||||
prefs = ('ffprobe', 'avprobe')
|
|
||||||
for p in prefs:
|
|
||||||
if self._versions[p]:
|
|
||||||
self.probe_basename = p
|
|
||||||
break
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def available(self):
|
def available(self):
|
||||||
|
|
|
@ -45,6 +45,7 @@ from .compat import (
|
||||||
compat_casefold,
|
compat_casefold,
|
||||||
compat_chr,
|
compat_chr,
|
||||||
compat_collections_abc,
|
compat_collections_abc,
|
||||||
|
compat_contextlib_suppress,
|
||||||
compat_cookiejar,
|
compat_cookiejar,
|
||||||
compat_ctypes_WINFUNCTYPE,
|
compat_ctypes_WINFUNCTYPE,
|
||||||
compat_datetime_timedelta_total_seconds,
|
compat_datetime_timedelta_total_seconds,
|
||||||
|
@ -1855,25 +1856,18 @@ def write_json_file(obj, fn):
|
||||||
try:
|
try:
|
||||||
with tf:
|
with tf:
|
||||||
json.dump(obj, tf)
|
json.dump(obj, tf)
|
||||||
if sys.platform == 'win32':
|
with compat_contextlib_suppress(OSError):
|
||||||
# Need to remove existing file on Windows, else os.rename raises
|
if sys.platform == 'win32':
|
||||||
# WindowsError or FileExistsError.
|
# Need to remove existing file on Windows, else os.rename raises
|
||||||
try:
|
# WindowsError or FileExistsError.
|
||||||
os.unlink(fn)
|
os.unlink(fn)
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
mask = os.umask(0)
|
mask = os.umask(0)
|
||||||
os.umask(mask)
|
os.umask(mask)
|
||||||
os.chmod(tf.name, 0o666 & ~mask)
|
os.chmod(tf.name, 0o666 & ~mask)
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
os.rename(tf.name, fn)
|
os.rename(tf.name, fn)
|
||||||
except Exception:
|
except Exception:
|
||||||
try:
|
with compat_contextlib_suppress(OSError):
|
||||||
os.remove(tf.name)
|
os.remove(tf.name)
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
@ -2033,14 +2027,13 @@ def extract_attributes(html_element):
|
||||||
NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
|
NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
|
||||||
but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
|
but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
|
||||||
"""
|
"""
|
||||||
parser = HTMLAttributeParser()
|
ret = None
|
||||||
try:
|
# Older Python may throw HTMLParseError in case of malformed HTML (and on .close()!)
|
||||||
parser.feed(html_element)
|
with compat_contextlib_suppress(compat_HTMLParseError):
|
||||||
parser.close()
|
with contextlib.closing(HTMLAttributeParser()) as parser:
|
||||||
# Older Python may throw HTMLParseError in case of malformed HTML
|
parser.feed(html_element)
|
||||||
except compat_HTMLParseError:
|
ret = parser.attrs
|
||||||
pass
|
return ret or {}
|
||||||
return parser.attrs
|
|
||||||
|
|
||||||
|
|
||||||
def clean_html(html):
|
def clean_html(html):
|
||||||
|
@ -2182,8 +2175,28 @@ def sanitize_url(url):
|
||||||
return url
|
return url
|
||||||
|
|
||||||
|
|
||||||
|
def extract_basic_auth(url):
|
||||||
|
parts = compat_urllib_parse.urlsplit(url)
|
||||||
|
if parts.username is None:
|
||||||
|
return url, None
|
||||||
|
url = compat_urllib_parse.urlunsplit(parts._replace(netloc=(
|
||||||
|
parts.hostname if parts.port is None
|
||||||
|
else '%s:%d' % (parts.hostname, parts.port))))
|
||||||
|
auth_payload = base64.b64encode(
|
||||||
|
('%s:%s' % (parts.username, parts.password or '')).encode('utf-8'))
|
||||||
|
return url, 'Basic {0}'.format(auth_payload.decode('ascii'))
|
||||||
|
|
||||||
|
|
||||||
def sanitized_Request(url, *args, **kwargs):
|
def sanitized_Request(url, *args, **kwargs):
|
||||||
return compat_urllib_request.Request(escape_url(sanitize_url(url)), *args, **kwargs)
|
url, auth_header = extract_basic_auth(escape_url(sanitize_url(url)))
|
||||||
|
if auth_header is not None:
|
||||||
|
headers = args[1] if len(args) > 1 else kwargs.get('headers')
|
||||||
|
headers = headers or {}
|
||||||
|
headers['Authorization'] = auth_header
|
||||||
|
if len(args) <= 1 and kwargs.get('headers') is None:
|
||||||
|
kwargs['headers'] = headers
|
||||||
|
kwargs = compat_kwargs(kwargs)
|
||||||
|
return compat_urllib_request.Request(url, *args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
def expand_path(s):
|
def expand_path(s):
|
||||||
|
@ -2221,7 +2234,8 @@ def _htmlentity_transform(entity_with_semicolon):
|
||||||
numstr = '0%s' % numstr
|
numstr = '0%s' % numstr
|
||||||
else:
|
else:
|
||||||
base = 10
|
base = 10
|
||||||
# See https://github.com/ytdl-org/youtube-dl/issues/7518
|
# See https://github.com/ytdl-org/youtube-dl/issues/7518\
|
||||||
|
# Also, weirdly, compat_contextlib_suppress fails here in 2.6
|
||||||
try:
|
try:
|
||||||
return compat_chr(int(numstr, base))
|
return compat_chr(int(numstr, base))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
|
@ -2328,11 +2342,9 @@ def make_HTTPS_handler(params, **kwargs):
|
||||||
# Some servers may (wrongly) reject requests if ALPN extension is not sent. See:
|
# Some servers may (wrongly) reject requests if ALPN extension is not sent. See:
|
||||||
# https://github.com/python/cpython/issues/85140
|
# https://github.com/python/cpython/issues/85140
|
||||||
# https://github.com/yt-dlp/yt-dlp/issues/3878
|
# https://github.com/yt-dlp/yt-dlp/issues/3878
|
||||||
try:
|
with compat_contextlib_suppress(AttributeError, NotImplementedError):
|
||||||
|
# fails for Python < 2.7.10, not ssl.HAS_ALPN
|
||||||
ctx.set_alpn_protocols(ALPN_PROTOCOLS)
|
ctx.set_alpn_protocols(ALPN_PROTOCOLS)
|
||||||
except (AttributeError, NotImplementedError):
|
|
||||||
# Python < 2.7.10, not ssl.HAS_ALPN
|
|
||||||
pass
|
|
||||||
|
|
||||||
opts_no_check_certificate = params.get('nocheckcertificate', False)
|
opts_no_check_certificate = params.get('nocheckcertificate', False)
|
||||||
if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
|
if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
|
||||||
|
@ -2342,12 +2354,10 @@ def make_HTTPS_handler(params, **kwargs):
|
||||||
context.check_hostname = False
|
context.check_hostname = False
|
||||||
context.verify_mode = ssl.CERT_NONE
|
context.verify_mode = ssl.CERT_NONE
|
||||||
|
|
||||||
try:
|
with compat_contextlib_suppress(TypeError):
|
||||||
|
# Fails with Python 2.7.8 (create_default_context present
|
||||||
|
# but HTTPSHandler has no context=)
|
||||||
return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
|
return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
|
||||||
except TypeError:
|
|
||||||
# Python 2.7.8
|
|
||||||
# (create_default_context present but HTTPSHandler has no context=)
|
|
||||||
pass
|
|
||||||
|
|
||||||
if sys.version_info < (3, 2):
|
if sys.version_info < (3, 2):
|
||||||
return YoutubeDLHTTPSHandler(params, **kwargs)
|
return YoutubeDLHTTPSHandler(params, **kwargs)
|
||||||
|
@ -2361,15 +2371,24 @@ def make_HTTPS_handler(params, **kwargs):
|
||||||
return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
|
return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
def bug_reports_message():
|
def bug_reports_message(before=';'):
|
||||||
if ytdl_is_updateable():
|
if ytdl_is_updateable():
|
||||||
update_cmd = 'type youtube-dl -U to update'
|
update_cmd = 'type youtube-dl -U to update'
|
||||||
else:
|
else:
|
||||||
update_cmd = 'see https://yt-dl.org/update on how to update'
|
update_cmd = 'see https://github.com/ytdl-org/youtube-dl/#user-content-installation on how to update'
|
||||||
msg = '; please report this issue on https://yt-dl.org/bug .'
|
|
||||||
msg += ' Make sure you are using the latest version; %s.' % update_cmd
|
msg = (
|
||||||
msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
|
'please report this issue on https://github.com/ytdl-org/youtube-dl/issues ,'
|
||||||
return msg
|
' using the appropriate issue template.'
|
||||||
|
' Make sure you are using the latest version; %s.'
|
||||||
|
' Be sure to call youtube-dl with the --verbose option and include the complete output.'
|
||||||
|
) % update_cmd
|
||||||
|
|
||||||
|
before = (before or '').rstrip()
|
||||||
|
if not before or before.endswith(('.', '!', '?')):
|
||||||
|
msg = msg[0].title() + msg[1:]
|
||||||
|
|
||||||
|
return (before + ' ' if before else '') + msg
|
||||||
|
|
||||||
|
|
||||||
class YoutubeDLError(Exception):
|
class YoutubeDLError(Exception):
|
||||||
|
@ -3156,12 +3175,10 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
|
||||||
if timezone is None:
|
if timezone is None:
|
||||||
timezone, date_str = extract_timezone(date_str)
|
timezone, date_str = extract_timezone(date_str)
|
||||||
|
|
||||||
try:
|
with compat_contextlib_suppress(ValueError):
|
||||||
date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
|
date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
|
||||||
dt = datetime.datetime.strptime(date_str, date_format) - timezone
|
dt = datetime.datetime.strptime(date_str, date_format) - timezone
|
||||||
return calendar.timegm(dt.timetuple())
|
return calendar.timegm(dt.timetuple())
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def date_formats(day_first=True):
|
def date_formats(day_first=True):
|
||||||
|
@ -3181,17 +3198,13 @@ def unified_strdate(date_str, day_first=True):
|
||||||
_, date_str = extract_timezone(date_str)
|
_, date_str = extract_timezone(date_str)
|
||||||
|
|
||||||
for expression in date_formats(day_first):
|
for expression in date_formats(day_first):
|
||||||
try:
|
with compat_contextlib_suppress(ValueError):
|
||||||
upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
|
upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
if upload_date is None:
|
if upload_date is None:
|
||||||
timetuple = email.utils.parsedate_tz(date_str)
|
timetuple = email.utils.parsedate_tz(date_str)
|
||||||
if timetuple:
|
if timetuple:
|
||||||
try:
|
with compat_contextlib_suppress(ValueError):
|
||||||
upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
|
upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
if upload_date is not None:
|
if upload_date is not None:
|
||||||
return compat_str(upload_date)
|
return compat_str(upload_date)
|
||||||
|
|
||||||
|
@ -3220,11 +3233,9 @@ def unified_timestamp(date_str, day_first=True):
|
||||||
date_str = m.group(1)
|
date_str = m.group(1)
|
||||||
|
|
||||||
for expression in date_formats(day_first):
|
for expression in date_formats(day_first):
|
||||||
try:
|
with compat_contextlib_suppress(ValueError):
|
||||||
dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
|
dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
|
||||||
return calendar.timegm(dt.timetuple())
|
return calendar.timegm(dt.timetuple())
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
timetuple = email.utils.parsedate_tz(date_str)
|
timetuple = email.utils.parsedate_tz(date_str)
|
||||||
if timetuple:
|
if timetuple:
|
||||||
return calendar.timegm(timetuple) + pm_delta * 3600 - compat_datetime_timedelta_total_seconds(timezone)
|
return calendar.timegm(timetuple) + pm_delta * 3600 - compat_datetime_timedelta_total_seconds(timezone)
|
||||||
|
@ -3832,14 +3843,15 @@ class PUTRequest(compat_urllib_request.Request):
|
||||||
return 'PUT'
|
return 'PUT'
|
||||||
|
|
||||||
|
|
||||||
def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
|
def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1, base=None):
|
||||||
if get_attr:
|
if get_attr:
|
||||||
if v is not None:
|
if v is not None:
|
||||||
v = getattr(v, get_attr, None)
|
v = getattr(v, get_attr, None)
|
||||||
if v in (None, ''):
|
if v in (None, ''):
|
||||||
return default
|
return default
|
||||||
try:
|
try:
|
||||||
return int(v) * invscale // scale
|
# like int, raise if base is specified and v is not a string
|
||||||
|
return (int(v) if base is None else int(v, base=base)) * invscale // scale
|
||||||
except (ValueError, TypeError, OverflowError):
|
except (ValueError, TypeError, OverflowError):
|
||||||
return default
|
return default
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue