From 8af49fc276b2cf2154b9342de4b4cd66f9d17af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 23 May 2019 23:48:06 +0700 Subject: [PATCH 1/7] [pornflip] Remove extractor --- youtube_dl/extractor/extractors.py | 1 - youtube_dl/extractor/pornflip.py | 101 ----------------------------- 2 files changed, 102 deletions(-) delete mode 100644 youtube_dl/extractor/pornflip.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 7705f9bdd..eb5efd1e8 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -888,7 +888,6 @@ from .polskieradio import ( from .popcorntv import PopcornTVIE from .porn91 import Porn91IE from .porncom import PornComIE -from .pornflip import PornFlipIE from .pornhd import PornHdIE from .pornhub import ( PornHubIE, diff --git a/youtube_dl/extractor/pornflip.py b/youtube_dl/extractor/pornflip.py deleted file mode 100644 index 025985fbc..000000000 --- a/youtube_dl/extractor/pornflip.py +++ /dev/null @@ -1,101 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .common import InfoExtractor -from ..compat import ( - compat_parse_qs, - compat_str, -) -from ..utils import ( - int_or_none, - try_get, - unified_timestamp, -) - - -class PornFlipIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P[^/?#&]+)' - _TESTS = [{ - 'url': 'https://www.pornflip.com/v/wz7DfNhMmep', - 'md5': '98c46639849145ae1fd77af532a9278c', - 'info_dict': { - 'id': 'wz7DfNhMmep', - 'ext': 'mp4', - 'title': '2 Amateurs swallow make his dream cumshots true', - 'thumbnail': r're:^https?://.*\.jpg$', - 'duration': 112, - 'timestamp': 1481655502, - 'upload_date': '20161213', - 'uploader_id': '106786', - 'uploader': 'figifoto', - 'view_count': int, - 'age_limit': 18, - } - }, { - 'url': 'https://www.pornflip.com/embed/wz7DfNhMmep', - 'only_matching': True, - }, { - 'url': 'https://www.pornflip.com/v/EkRD6-vS2-s', - 'only_matching': True, - }, { - 'url': 'https://www.pornflip.com/embed/EkRD6-vS2-s', - 'only_matching': True, - }, { - 'url': 'https://www.pornflip.com/v/NG9q6Pb_iK8', - 'only_matching': True, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - - webpage = self._download_webpage( - 'https://www.pornflip.com/v/%s' % video_id, video_id) - - flashvars = compat_parse_qs(self._search_regex( - r']+flashvars=(["\'])(?P(?:(?!\1).)+)\1', - webpage, 'flashvars', group='flashvars')) - - title = flashvars['video_vars[title]'][0] - - def flashvar(kind): - return try_get( - flashvars, lambda x: x['video_vars[%s]' % kind][0], compat_str) - - formats = [] - for key, value in flashvars.items(): - if not (value and isinstance(value, list)): - continue - format_url = value[0] - if key == 'video_vars[hds_manifest]': - formats.extend(self._extract_mpd_formats( - format_url, video_id, mpd_id='dash', fatal=False)) - continue - height = self._search_regex( - r'video_vars\[video_urls\]\[(\d+)', key, 'height', default=None) - if not height: - continue - formats.append({ - 'url': format_url, - 'format_id': 'http-%s' % height, - 'height': int_or_none(height), - }) - self._sort_formats(formats) - - uploader = self._html_search_regex( - (r']+class="name"[^>]*>\s*]+>\s*(?P[^<]+)', - r']+content=(["\'])[^>]*\buploaded by (?P.+?)\1'), - webpage, 'uploader', fatal=False, group='uploader') - - return { - 'id': video_id, - 'formats': formats, - 'title': title, - 'thumbnail': flashvar('big_thumb'), - 'duration': int_or_none(flashvar('duration')), - 'timestamp': unified_timestamp(self._html_search_meta( - 'uploadDate', webpage, 'timestamp')), - 'uploader_id': flashvar('author_id'), - 'uploader': uploader, - 'view_count': int_or_none(flashvar('views')), - 'age_limit': 18, - } From f856816b940d95e995c2b9995d01097ab144a1af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 23 May 2019 23:52:11 +0700 Subject: [PATCH 2/7] [extractor/common] Strip src attribute for HTML5 entries code (closes #18485, closes #21169) --- youtube_dl/extractor/common.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 937237b3f..9c3e9eec6 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -67,6 +67,7 @@ from ..utils import ( sanitized_Request, sanitize_filename, str_or_none, + strip_or_none, unescapeHTML, unified_strdate, unified_timestamp, @@ -2480,7 +2481,7 @@ class InfoExtractor(object): 'subtitles': {}, } media_attributes = extract_attributes(media_tag) - src = media_attributes.get('src') + src = strip_or_none(media_attributes.get('src')) if src: _, formats = _media_formats(src, media_type) media_info['formats'].extend(formats) @@ -2490,7 +2491,7 @@ class InfoExtractor(object): s_attr = extract_attributes(source_tag) # data-video-src and data-src are non standard but seen # several times in the wild - src = dict_get(s_attr, ('src', 'data-video-src', 'data-src')) + src = strip_or_none(dict_get(s_attr, ('src', 'data-video-src', 'data-src'))) if not src: continue f = parse_content_type(s_attr.get('type')) @@ -2533,7 +2534,7 @@ class InfoExtractor(object): track_attributes = extract_attributes(track_tag) kind = track_attributes.get('kind') if not kind or kind in ('subtitles', 'captions'): - src = track_attributes.get('src') + src = strip_or_none(track_attributes.get('src')) if not src: continue lang = track_attributes.get('srclang') or track_attributes.get('lang') or track_attributes.get('label') From 53cd37bac50e7a927deba5b67a4412301b96230d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 23 May 2019 23:58:35 +0700 Subject: [PATCH 3/7] [utils] Improve strip_or_none --- test/test_utils.py | 13 +++++++++++++ youtube_dl/utils.py | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index 9ef0e422b..71980b3fc 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -73,6 +73,7 @@ from youtube_dl.utils import ( smuggle_url, str_to_int, strip_jsonp, + strip_or_none, timeconvert, unescapeHTML, unified_strdate, @@ -752,6 +753,18 @@ class TestUtil(unittest.TestCase): d = json.loads(stripped) self.assertEqual(d, {'status': 'success'}) + def test_strip_or_none(self): + self.assertEqual(strip_or_none(' abc'), 'abc') + self.assertEqual(strip_or_none('abc '), 'abc') + self.assertEqual(strip_or_none(' abc '), 'abc') + self.assertEqual(strip_or_none('\tabc\t'), 'abc') + self.assertEqual(strip_or_none('\n\tabc\n\t'), 'abc') + self.assertEqual(strip_or_none('abc'), 'abc') + self.assertEqual(strip_or_none(''), '') + self.assertEqual(strip_or_none(None), None) + self.assertEqual(strip_or_none(42), None) + self.assertEqual(strip_or_none([]), None) + def test_uppercase_escape(self): self.assertEqual(uppercase_escape('aä'), 'aä') self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 9be9b2e76..ead9bd862 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1951,8 +1951,8 @@ def bool_or_none(v, default=None): return v if isinstance(v, bool) else default -def strip_or_none(v): - return None if v is None else v.strip() +def strip_or_none(v, default=None): + return v.strip() if isinstance(v, compat_str) else default def url_or_none(url): From 11ec06de7f01bced1f4f4b6484e5190cbb9ed9e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 24 May 2019 00:35:46 +0700 Subject: [PATCH 4/7] [24video] Add support for 24video.site (closes #21193) --- youtube_dl/extractor/twentyfourvideo.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py index 4b3b3e705..d16f55500 100644 --- a/youtube_dl/extractor/twentyfourvideo.py +++ b/youtube_dl/extractor/twentyfourvideo.py @@ -14,7 +14,7 @@ from ..utils import ( class TwentyFourVideoIE(InfoExtractor): IE_NAME = '24video' - _VALID_URL = r'https?://(?P(?:www\.)?24video\.(?:net|me|xxx|sexy?|tube|adult))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P\d+)' + _VALID_URL = r'https?://(?P(?:www\.)?24video\.(?:net|me|xxx|sexy?|tube|adult|site))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P\d+)' _TESTS = [{ 'url': 'http://www.24video.net/video/view/1044982', @@ -42,6 +42,9 @@ class TwentyFourVideoIE(InfoExtractor): }, { 'url': 'http://www.24video.tube/video/view/2363750', 'only_matching': True, + }, { + 'url': 'https://www.24video.site/video/view/2640421', + 'only_matching': True, }] def _real_extract(self, url): From f4cc2ca503239aadd5cb75f83cb873bc5816dfdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 24 May 2019 00:38:06 +0700 Subject: [PATCH 5/7] [24video] Add support for porno.24video.net (closes #21194) --- youtube_dl/extractor/twentyfourvideo.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py index d16f55500..1d66eeaff 100644 --- a/youtube_dl/extractor/twentyfourvideo.py +++ b/youtube_dl/extractor/twentyfourvideo.py @@ -14,7 +14,18 @@ from ..utils import ( class TwentyFourVideoIE(InfoExtractor): IE_NAME = '24video' - _VALID_URL = r'https?://(?P(?:www\.)?24video\.(?:net|me|xxx|sexy?|tube|adult|site))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P\d+)' + _VALID_URL = r'''(?x) + https?:// + (?P + (?:(?:www|porno)\.)?24video\. + (?:net|me|xxx|sexy?|tube|adult|site) + )/ + (?: + video/(?:(?:view|xml)/)?| + player/new24_play\.swf\?id= + ) + (?P\d+) + ''' _TESTS = [{ 'url': 'http://www.24video.net/video/view/1044982', @@ -45,6 +56,9 @@ class TwentyFourVideoIE(InfoExtractor): }, { 'url': 'https://www.24video.site/video/view/2640421', 'only_matching': True, + }, { + 'url': 'https://porno.24video.net/video/2640421-vsya-takaya-gibkaya-i-v-masle', + 'only_matching': True, }] def _real_extract(self, url): From 3fe774722b0855d572b6885ff20fb6f4f96f3e08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20P=C3=B6schel?= Date: Sun, 19 May 2019 18:11:17 +0200 Subject: [PATCH 6/7] [srgssrplay] Add support for popupvideoplayer URLs --- youtube_dl/extractor/srgssr.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py index bb73eb1d5..ff1b67876 100644 --- a/youtube_dl/extractor/srgssr.py +++ b/youtube_dl/extractor/srgssr.py @@ -106,7 +106,7 @@ class SRGSSRIE(InfoExtractor): class SRGSSRPlayIE(InfoExtractor): IE_DESC = 'srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites' - _VALID_URL = r'https?://(?:(?:www|play)\.)?(?Psrf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/[^/]+/(?Pvideo|audio)/[^?]+\?id=(?P[0-9a-f\-]{36}|\d+)' + _VALID_URL = r'https?://(?:(?:www|play)\.)?(?Psrf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/(?:[^/]+/|popup)(?Pvideo|audio)(?:/[^?]+|player)\?id=(?P[0-9a-f\-]{36}|\d+)' _TESTS = [{ 'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5', @@ -163,6 +163,17 @@ class SRGSSRPlayIE(InfoExtractor): # m3u8 download 'skip_download': True, } + }, { + 'url': 'https://www.srf.ch/play/tv/popupvideoplayer?id=c4dba0ca-e75b-43b2-a34f-f708a4932e01', + 'md5': 'f6247aa7c905b81c9ba7f50fb22e2fbd', + 'info_dict': { + 'id': 'c4dba0ca-e75b-43b2-a34f-f708a4932e01', + 'ext': 'mp4', + 'upload_date': '20190122', + 'title': 'Erster Selfie-Stick (1983)', + 'description': 'md5:23a6b40024e583137e4137f5946543c1', + 'timestamp': 1548155133, + } }] def _real_extract(self, url): From 25b83c2a0e29c75372f0ce26d2b4ecf493e8b28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 24 May 2019 00:43:22 +0700 Subject: [PATCH 7/7] [srgssrplay] Improve _VALID_URL (closes #21155) --- youtube_dl/extractor/srgssr.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py index ff1b67876..170dce87f 100644 --- a/youtube_dl/extractor/srgssr.py +++ b/youtube_dl/extractor/srgssr.py @@ -106,7 +106,16 @@ class SRGSSRIE(InfoExtractor): class SRGSSRPlayIE(InfoExtractor): IE_DESC = 'srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites' - _VALID_URL = r'https?://(?:(?:www|play)\.)?(?Psrf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/(?:[^/]+/|popup)(?Pvideo|audio)(?:/[^?]+|player)\?id=(?P[0-9a-f\-]{36}|\d+)' + _VALID_URL = r'''(?x) + https?:// + (?:(?:www|play)\.)? + (?Psrf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/ + (?: + [^/]+/(?Pvideo|audio)/[^?]+| + popup(?Pvideo|audio)player + ) + \?id=(?P[0-9a-f\-]{36}|\d+) + ''' _TESTS = [{ 'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5', @@ -165,18 +174,13 @@ class SRGSSRPlayIE(InfoExtractor): } }, { 'url': 'https://www.srf.ch/play/tv/popupvideoplayer?id=c4dba0ca-e75b-43b2-a34f-f708a4932e01', - 'md5': 'f6247aa7c905b81c9ba7f50fb22e2fbd', - 'info_dict': { - 'id': 'c4dba0ca-e75b-43b2-a34f-f708a4932e01', - 'ext': 'mp4', - 'upload_date': '20190122', - 'title': 'Erster Selfie-Stick (1983)', - 'description': 'md5:23a6b40024e583137e4137f5946543c1', - 'timestamp': 1548155133, - } + 'only_matching': True, }] def _real_extract(self, url): - bu, media_type, media_id = re.match(self._VALID_URL, url).groups() + mobj = re.match(self._VALID_URL, url) + bu = mobj.group('bu') + media_type = mobj.group('type') or mobj.group('type_2') + media_id = mobj.group('id') # other info can be extracted from url + '&layout=json' return self.url_result('srgssr:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')