Merge branch 'master' of github.com:rg3/youtube-dl

* 'master' of github.com:rg3/youtube-dl:
  [srgssrplay] Improve _VALID_URL (closes #21155)
  [srgssrplay] Add support for popupvideoplayer URLs
  [24video] Add support for porno.24video.net (closes #21194)
  [24video] Add support for 24video.site (closes #21193)
  [utils] Improve strip_or_none
  [extractor/common] Strip src attribute for HTML5 entries code (closes #18485, closes #21169)
  [pornflip] Remove extractor

test/test_utils.py

@@ -73,6 +73,7 @@ from youtube_dl.utils import (
     smuggle_url,
     str_to_int,
     strip_jsonp,
+    strip_or_none,
     timeconvert,
     unescapeHTML,
     unified_strdate,
@@ -752,6 +753,18 @@ class TestUtil(unittest.TestCase):
         d = json.loads(stripped)
         self.assertEqual(d, {'status': 'success'})
 
+    def test_strip_or_none(self):
+        self.assertEqual(strip_or_none(' abc'), 'abc')
+        self.assertEqual(strip_or_none('abc '), 'abc')
+        self.assertEqual(strip_or_none(' abc '), 'abc')
+        self.assertEqual(strip_or_none('\tabc\t'), 'abc')
+        self.assertEqual(strip_or_none('\n\tabc\n\t'), 'abc')
+        self.assertEqual(strip_or_none('abc'), 'abc')
+        self.assertEqual(strip_or_none(''), '')
+        self.assertEqual(strip_or_none(None), None)
+        self.assertEqual(strip_or_none(42), None)
+        self.assertEqual(strip_or_none([]), None)
+
     def test_uppercase_escape(self):
         self.assertEqual(uppercase_escape('aä'), 'aä')
         self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')

youtube_dl/extractor/common.py

@@ -67,6 +67,7 @@ from ..utils import (
     sanitized_Request,
     sanitize_filename,
     str_or_none,
+    strip_or_none,
     unescapeHTML,
     unified_strdate,
     unified_timestamp,
@@ -2480,7 +2481,7 @@ class InfoExtractor(object):
                 'subtitles': {},
             }
             media_attributes = extract_attributes(media_tag)
-            src = media_attributes.get('src')
+            src = strip_or_none(media_attributes.get('src'))
             if src:
                 _, formats = _media_formats(src, media_type)
                 media_info['formats'].extend(formats)
@@ -2490,7 +2491,7 @@ class InfoExtractor(object):
                     s_attr = extract_attributes(source_tag)
                     # data-video-src and data-src are non standard but seen
                     # several times in the wild
-                    src = dict_get(s_attr, ('src', 'data-video-src', 'data-src'))
+                    src = strip_or_none(dict_get(s_attr, ('src', 'data-video-src', 'data-src')))
                     if not src:
                         continue
                     f = parse_content_type(s_attr.get('type'))
@@ -2533,7 +2534,7 @@ class InfoExtractor(object):
                     track_attributes = extract_attributes(track_tag)
                     kind = track_attributes.get('kind')
                     if not kind or kind in ('subtitles', 'captions'):
-                        src = track_attributes.get('src')
+                        src = strip_or_none(track_attributes.get('src'))
                         if not src:
                             continue
                         lang = track_attributes.get('srclang') or track_attributes.get('lang') or track_attributes.get('label')

youtube_dl/extractor/extractors.py

@@ -888,7 +888,6 @@ from .polskieradio import (
 from .popcorntv import PopcornTVIE
 from .porn91 import Porn91IE
 from .porncom import PornComIE
-from .pornflip import PornFlipIE
 from .pornhd import PornHdIE
 from .pornhub import (
     PornHubIE,

youtube_dl/extractor/pornflip.py

@@ -1,101 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_parse_qs,
-    compat_str,
-)
-from ..utils import (
-    int_or_none,
-    try_get,
-    unified_timestamp,
-)
-
-
-class PornFlipIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'https://www.pornflip.com/v/wz7DfNhMmep',
-        'md5': '98c46639849145ae1fd77af532a9278c',
-        'info_dict': {
-            'id': 'wz7DfNhMmep',
-            'ext': 'mp4',
-            'title': '2 Amateurs swallow make his dream cumshots true',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 112,
-            'timestamp': 1481655502,
-            'upload_date': '20161213',
-            'uploader_id': '106786',
-            'uploader': 'figifoto',
-            'view_count': int,
-            'age_limit': 18,
-        }
-    }, {
-        'url': 'https://www.pornflip.com/embed/wz7DfNhMmep',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.pornflip.com/v/EkRD6-vS2-s',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.pornflip.com/embed/EkRD6-vS2-s',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.pornflip.com/v/NG9q6Pb_iK8',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'https://www.pornflip.com/v/%s' % video_id, video_id)
-
-        flashvars = compat_parse_qs(self._search_regex(
-            r'<embed[^>]+flashvars=(["\'])(?P<flashvars>(?:(?!\1).)+)\1',
-            webpage, 'flashvars', group='flashvars'))
-
-        title = flashvars['video_vars[title]'][0]
-
-        def flashvar(kind):
-            return try_get(
-                flashvars, lambda x: x['video_vars[%s]' % kind][0], compat_str)
-
-        formats = []
-        for key, value in flashvars.items():
-            if not (value and isinstance(value, list)):
-                continue
-            format_url = value[0]
-            if key == 'video_vars[hds_manifest]':
-                formats.extend(self._extract_mpd_formats(
-                    format_url, video_id, mpd_id='dash', fatal=False))
-                continue
-            height = self._search_regex(
-                r'video_vars\[video_urls\]\[(\d+)', key, 'height', default=None)
-            if not height:
-                continue
-            formats.append({
-                'url': format_url,
-                'format_id': 'http-%s' % height,
-                'height': int_or_none(height),
-            })
-        self._sort_formats(formats)
-
-        uploader = self._html_search_regex(
-            (r'<span[^>]+class="name"[^>]*>\s*<a[^>]+>\s*<strong>(?P<uploader>[^<]+)',
-             r'<meta[^>]+content=(["\'])[^>]*\buploaded by (?P<uploader>.+?)\1'),
-            webpage, 'uploader', fatal=False, group='uploader')
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'title': title,
-            'thumbnail': flashvar('big_thumb'),
-            'duration': int_or_none(flashvar('duration')),
-            'timestamp': unified_timestamp(self._html_search_meta(
-                'uploadDate', webpage, 'timestamp')),
-            'uploader_id': flashvar('author_id'),
-            'uploader': uploader,
-            'view_count': int_or_none(flashvar('views')),
-            'age_limit': 18,
-        }

youtube_dl/extractor/srgssr.py

@@ -106,7 +106,16 @@ class SRGSSRIE(InfoExtractor):
 
 class SRGSSRPlayIE(InfoExtractor):
     IE_DESC = 'srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites'
-    _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/[^/]+/(?P<type>video|audio)/[^?]+\?id=(?P<id>[0-9a-f\-]{36}|\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:(?:www|play)\.)?
+                        (?P<bu>srf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/
+                        (?:
+                            [^/]+/(?P<type>video|audio)/[^?]+|
+                            popup(?P<type_2>video|audio)player
+                        )
+                        \?id=(?P<id>[0-9a-f\-]{36}|\d+)
+                    '''
 
     _TESTS = [{
         'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
@@ -163,9 +172,15 @@ class SRGSSRPlayIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         }
+    }, {
+        'url': 'https://www.srf.ch/play/tv/popupvideoplayer?id=c4dba0ca-e75b-43b2-a34f-f708a4932e01',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
+        mobj = re.match(self._VALID_URL, url)
+        bu = mobj.group('bu')
+        media_type = mobj.group('type') or mobj.group('type_2')
+        media_id = mobj.group('id')
         # other info can be extracted from url + '&layout=json'
         return self.url_result('srgssr:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')

youtube_dl/extractor/twentyfourvideo.py

@@ -14,7 +14,18 @@ from ..utils import (
 
 class TwentyFourVideoIE(InfoExtractor):
     IE_NAME = '24video'
-    _VALID_URL = r'https?://(?P<host>(?:www\.)?24video\.(?:net|me|xxx|sexy?|tube|adult))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?P<host>
+                            (?:(?:www|porno)\.)?24video\.
+                            (?:net|me|xxx|sexy?|tube|adult|site)
+                        )/
+                        (?:
+                            video/(?:(?:view|xml)/)?|
+                            player/new24_play\.swf\?id=
+                        )
+                        (?P<id>\d+)
+                    '''
 
     _TESTS = [{
         'url': 'http://www.24video.net/video/view/1044982',
@@ -42,6 +53,12 @@ class TwentyFourVideoIE(InfoExtractor):
     }, {
         'url': 'http://www.24video.tube/video/view/2363750',
         'only_matching': True,
+    }, {
+        'url': 'https://www.24video.site/video/view/2640421',
+        'only_matching': True,
+    }, {
+        'url': 'https://porno.24video.net/video/2640421-vsya-takaya-gibkaya-i-v-masle',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

youtube_dl/utils.py

@@ -1951,8 +1951,8 @@ def bool_or_none(v, default=None):
     return v if isinstance(v, bool) else default
 
 
-def strip_or_none(v):
-    return None if v is None else v.strip()
+def strip_or_none(v, default=None):
+    return v.strip() if isinstance(v, compat_str) else default
 
 
 def url_or_none(url):