Add --socks-proxy parameter to use SOCKS proxy

This allows the user to specify a socks proxy to
tunnel the connection through. This feature requires that
PySocks is available. If the user tries to use a SOCKS proxy
without PySocks available an error is printed and the process is
aborted.

youtube_dl/YoutubeDL.py

@@ -59,6 +59,7 @@ from .utils import (
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,
+    OptionalDependencyNotFound,
     PagedList,
     parse_filesize,
     PerRequestProxyHandler,
@@ -1831,32 +1832,39 @@ class YoutubeDL(object):
                 proxies['https'] = proxies['http']
         proxy_handler = PerRequestProxyHandler(proxies)
 
-        if 'socks' in sys.modules:  # Check if socks was imported
-            opts_socks = self.params.get('socksproxy')
+        socks_handler = None
+        opts_socks = self.params.get('socksproxy')
+        if opts_socks is not None and opts_socks:
+            # Try to import the dependencies for this feature
+            try:
+                import socks
+            except ImportError:
+                raise OptionalDependencyNotFound(module_name='socks',
+                                                 feature_name='"SocksProxy"')
+            try:
+                from sockshandler import SocksiPyHandler
+            except ImportError:
+                raise OptionalDependencyNotFound(module_name='sockshandler',
+                                                 feature_name='"SocksProxy"')
 
-            if opts_socks is not None and opts_socks:
-                pair = opts_socks.split(':')
-                if len(pair) == 2:
-                    socks.setdefaultproxy(
-                        socks.PROXY_TYPE_SOCKS5,
-                        pair[0],
-                        int(pair[1]))
-                else:
-                    socks.setdefaultproxy(
-                        socks.PROXY_TYPE_SOCKS5,
-                        'localhost',
-                        int(pair[0]))
-
-                socks.wrapmodule(compat_urllib_request)
-        else:  # Socks was not imported, but the use tried to use it. Tell them
-            if self.params.get('socksproxy'):
-                self.report_warning("Can't use socks proxy, socks module not found")
+            pair = opts_socks.split(':')
+            if len(pair) == 2:
+                socks_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS5,
+                                                pair[0],
+                                                int(pair[1]))
+            else:
+                socks_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS5,
+                                                'localhost',
+                                                int(pair[0]))
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
-        opener = compat_urllib_request.build_opener(
-            proxy_handler, https_handler, cookie_processor, ydlh)
+        proxy_list = []
+        if socks_handler:
+            proxy_list.append(socks_handler)
+        proxy_list += [proxy_handler, https_handler, cookie_processor, ydlh]
+        opener = compat_urllib_request.build_opener(*proxy_list)
 
         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play

youtube_dl/__init__.py

@@ -29,6 +29,7 @@ from .utils import (
     DownloadError,
     match_filter_func,
     MaxDownloadsReached,
+    OptionalDependencyNotFound,
     preferredencoding,
     read_batch_urls,
     SameFileError,
@@ -415,5 +416,7 @@ def main(argv=None):
         sys.exit('ERROR: fixed output name but more than one file to download')
     except KeyboardInterrupt:
         sys.exit('\nERROR: Interrupted by user')
+    except OptionalDependencyNotFound:
+        sys.exit('ERROR: Dependency not found')
 
 __all__ = ['main', 'YoutubeDL', 'gen_extractors', 'list_extractors']

youtube_dl/extractor/__init__.py

@@ -43,7 +43,10 @@ from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbccouk import BBCCoUkIE
+from .bbc import (
+    BBCCoUkIE,
+    BBCIE,
+)
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
@@ -243,6 +246,7 @@ from .instagram import InstagramIE, InstagramUserIE
 from .internetvideoarchive import InternetVideoArchiveIE
 from .iprima import IPrimaIE
 from .iqiyi import IqiyiIE
+from .ir90tv import Ir90TvIE
 from .ivi import (
     IviIE,
     IviCompilationIE

youtube_dl/extractor/bbc.py

@@ -1,11 +1,16 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
+import re
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    float_or_none,
     int_or_none,
+    parse_duration,
+    parse_iso8601,
 )
 from ..compat import compat_HTTPError
 
@@ -15,6 +20,8 @@ class BBCCoUkIE(InfoExtractor):
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
+    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
+
     _TESTS = [
         {
             'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
@@ -165,6 +172,7 @@ class BBCCoUkIE(InfoExtractor):
         supplier = connection.get('supplier')
         if protocol == 'http':
             href = connection.get('href')
+            transfer_format = connection.get('transferFormat')
             # ASX playlist
             if supplier == 'asx':
                 for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
@@ -172,6 +180,9 @@ class BBCCoUkIE(InfoExtractor):
                         'url': ref,
                         'format_id': 'ref%s_%s' % (i, supplier),
                     })
+            # Skip DASH until supported
+            elif transfer_format == 'dash':
+                pass
             # Direct link
             else:
                 formats.append({
@@ -260,16 +271,21 @@ class BBCCoUkIE(InfoExtractor):
         return subtitles
 
     def _download_media_selector(self, programme_id):
+        return self._download_media_selector_url(
+            self._MEDIASELECTOR_URL % programme_id, programme_id)
+
+    def _download_media_selector_url(self, url, programme_id=None):
         try:
             media_selection = self._download_xml(
-                'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' % programme_id,
-                programme_id, 'Downloading media selection XML')
+                url, programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
                 media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().decode('utf-8'))
             else:
                 raise
+        return self._process_media_selector(media_selection, programme_id)
 
+    def _process_media_selector(self, media_selection, programme_id):
         formats = []
         subtitles = None
 
@@ -308,10 +324,21 @@ class BBCCoUkIE(InfoExtractor):
                 raise
 
         # fallback to legacy playlist
-        playlist = self._download_xml(
-            'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id,
-            playlist_id, 'Downloading legacy playlist XML')
+        return self._process_legacy_playlist(playlist_id)
 
+    def _process_legacy_playlist_url(self, url, display_id):
+        playlist = self._download_legacy_playlist_url(url, display_id)
+        return self._extract_from_legacy_playlist(playlist, display_id)
+
+    def _process_legacy_playlist(self, playlist_id):
+        return self._process_legacy_playlist_url(
+            'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id, playlist_id)
+
+    def _download_legacy_playlist_url(self, url, playlist_id=None):
+        return self._download_xml(
+            url, playlist_id, 'Downloading legacy playlist XML')
+
+    def _extract_from_legacy_playlist(self, playlist, playlist_id):
         no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
         if no_items is not None:
             reason = no_items.get('reason')
@@ -331,8 +358,23 @@ class BBCCoUkIE(InfoExtractor):
                 continue
             title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
             description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
-            programme_id = item.get('identifier')
+
+            def get_programme_id(item):
+                def get_from_attributes(item):
+                    for p in('identifier', 'group'):
+                        value = item.get(p)
+                        if value and re.match(r'^[pb][\da-z]{7}$', value):
+                            return value
+                get_from_attributes(item)
+                mediator = item.find('./{http://bbc.co.uk/2008/emp/playlist}mediator')
+                if mediator is not None:
+                    return get_from_attributes(mediator)
+
+            programme_id = get_programme_id(item)
             duration = int_or_none(item.get('duration'))
+            # TODO: programme_id can be None and media items can be incorporated right inside
+            # playlist's item (e.g. http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
+            # as f4m and m3u8
             formats, subtitles = self._download_media_selector(programme_id)
 
         return programme_id, title, description, duration, formats, subtitles
@@ -377,3 +419,278 @@ class BBCCoUkIE(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class BBCIE(BBCCoUkIE):
+    IE_NAME = 'bbc'
+    IE_DESC = 'BBC'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
+
+    # fails with notukerror for some videos
+    # _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
+
+    _TESTS = [{
+        # article with multiple videos embedded with data-media-meta containing
+        # playlist.sxml, externalId and no direct video links
+        'url': 'http://www.bbc.com/news/world-europe-32668511',
+        'info_dict': {
+            'id': 'world-europe-32668511',
+            'title': 'Russia stages massive WW2 parade despite Western boycott',
+            'description': 'md5:00ff61976f6081841f759a08bf78cc9c',
+        },
+        'playlist_count': 2,
+    }, {
+        # article with multiple videos embedded with data-media-meta (more videos)
+        'url': 'http://www.bbc.com/news/business-28299555',
+        'info_dict': {
+            'id': 'business-28299555',
+            'title': 'Farnborough Airshow: Video highlights',
+            'description': 'BBC reports and video highlights at the Farnborough Airshow.',
+        },
+        'playlist_count': 9,
+        'skip': 'Save time',
+    }, {
+        # single video embedded with mediaAssetPage.init()
+        'url': 'http://www.bbc.com/news/world-europe-32041533',
+        'info_dict': {
+            'id': 'p02mprgb',
+            'ext': 'flv',
+            'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'duration': 47,
+            'timestamp': 1427219242,
+            'upload_date': '20150324',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # article with single video embedded with data-media-meta containing
+        # direct video links (for now these are extracted) and playlist.xml (with
+        # media items as f4m and m3u8 - currently unsupported)
+        'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
+        'info_dict': {
+            'id': '150615_telabyad_kentin_cogu',
+            'ext': 'mp4',
+            'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
+            'duration': 47,
+            'timestamp': 1434397334,
+            'upload_date': '20150615',
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        # single video embedded with mediaAssetPage.init() (regional section)
+        'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
+        'info_dict': {
+            'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
+            'ext': 'mp4',
+            'title': 'Honduras militariza sus hospitales por nuevo escándalo de corrupción',
+            'duration': 87,
+            'timestamp': 1434713142,
+            'upload_date': '20150619',
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        # single video story with digitalData
+        'url': 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret',
+        'info_dict': {
+            'id': 'p02q6gc4',
+            'ext': 'flv',
+            'title': 'Sri Lanka’s spicy secret',
+            'description': 'As a new train line to Jaffna opens up the country’s north, travellers can experience a truly distinct slice of Tamil culture.',
+            'timestamp': 1437674293,
+            'upload_date': '20150723',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video story without digitalData
+        'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
+        'info_dict': {
+            'id': 'p018zqqg',
+            'ext': 'flv',
+            'title': 'Hyundai Santa Fe Sport: Rock star',
+            'description': 'md5:b042a26142c4154a6e472933cf20793d',
+            'timestamp': 1368473503,
+            'upload_date': '20130513',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video with playlist.sxml URL
+        'url': 'http://www.bbc.com/sport/0/football/33653409',
+        'info_dict': {
+            'id': 'p02xycnp',
+            'ext': 'flv',
+            'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
+            'description': 'md5:398fca0e2e701c609d726e034fa1fc89',
+            'duration': 140,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video with playlist URL from weather section
+        'url': 'http://www.bbc.com/weather/features/33601775',
+        'only_matching': True,
+    }, {
+        # custom redirection to www.bbc.com
+        'url': 'http://www.bbc.co.uk/news/science-environment-33661876',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if BBCCoUkIE.suitable(url) else super(BBCIE, cls).suitable(url)
+
+    def _extract_from_media_meta(self, media_meta, video_id):
+        # Direct links to media in media metadata (e.g.
+        # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
+        # TODO: there are also f4m and m3u8 streams incorporated in playlist.sxml
+        source_files = media_meta.get('sourceFiles')
+        if source_files:
+            return [{
+                'url': f['url'],
+                'format_id': format_id,
+                'ext': f.get('encoding'),
+                'tbr': float_or_none(f.get('bitrate'), 1000),
+                'filesize': int_or_none(f.get('filesize')),
+            } for format_id, f in source_files.items() if f.get('url')], []
+
+        programme_id = media_meta.get('externalId')
+        if programme_id:
+            return self._download_media_selector(programme_id)
+
+        # Process playlist.sxml as legacy playlist
+        href = media_meta.get('href')
+        if href:
+            playlist = self._download_legacy_playlist_url(href)
+            _, _, _, _, formats, subtitles = self._extract_from_legacy_playlist(playlist, video_id)
+            return formats, subtitles
+
+        return [], []
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        timestamp = parse_iso8601(self._search_regex(
+            [r'"datePublished":\s*"([^"]+)',
+             r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
+             r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
+            webpage, 'date', default=None))
+
+        # single video with playlist.sxml URL (e.g. http://www.bbc.com/sport/0/football/3365340ng)
+        playlist = self._search_regex(
+            r'<param[^>]+name="playlist"[^>]+value="([^"]+)"',
+            webpage, 'playlist', default=None)
+        if playlist:
+            programme_id, title, description, duration, formats, subtitles = \
+                self._process_legacy_playlist_url(playlist, playlist_id)
+            self._sort_formats(formats)
+            return {
+                'id': programme_id,
+                'title': title,
+                'description': description,
+                'duration': duration,
+                'timestamp': timestamp,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
+        # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
+        programme_id = self._search_regex(
+            [r'data-video-player-vpid="([\da-z]{8})"',
+             r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
+            webpage, 'vpid', default=None)
+        if programme_id:
+            formats, subtitles = self._download_media_selector(programme_id)
+            self._sort_formats(formats)
+            # digitalData may be missing (e.g. http://www.bbc.com/autos/story/20130513-hyundais-rock-star)
+            digital_data = self._parse_json(
+                self._search_regex(
+                    r'var\s+digitalData\s*=\s*({.+?});?\n', webpage, 'digital data', default='{}'),
+                programme_id, fatal=False)
+            page_info = digital_data.get('page', {}).get('pageInfo', {})
+            title = page_info.get('pageName') or self._og_search_title(webpage)
+            description = page_info.get('description') or self._og_search_description(webpage)
+            timestamp = parse_iso8601(page_info.get('publicationDate')) or timestamp
+            return {
+                'id': programme_id,
+                'title': title,
+                'description': description,
+                'timestamp': timestamp,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
+        playlist_title = self._html_search_regex(
+            r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'playlist title')
+        playlist_description = self._og_search_description(webpage)
+
+        # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
+        medias = list(filter(None, map(
+            lambda s: self._parse_json(s, playlist_id, fatal=False),
+            re.findall(r"data-media-meta='({[^']+})'", webpage))))
+
+        if not medias:
+            # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
+            media_asset_page = self._parse_json(
+                self._search_regex(
+                    r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'media asset'),
+                playlist_id)
+            medias = []
+            for video in media_asset_page.get('videos', {}).values():
+                medias.extend(video.values())
+
+        entries = []
+        for num, media_meta in enumerate(medias, start=1):
+            formats, subtitles = self._extract_from_media_meta(media_meta, playlist_id)
+            if not formats:
+                continue
+            self._sort_formats(formats)
+
+            video_id = media_meta.get('externalId')
+            if not video_id:
+                video_id = playlist_id if len(medias) == 1 else '%s-%s' % (playlist_id, num)
+
+            title = media_meta.get('caption')
+            if not title:
+                title = playlist_title if len(medias) == 1 else '%s - Video %s' % (playlist_title, num)
+
+            duration = int_or_none(media_meta.get('durationInSeconds')) or parse_duration(media_meta.get('duration'))
+
+            images = []
+            for image in media_meta.get('images', {}).values():
+                images.extend(image.values())
+            if 'image' in media_meta:
+                images.append(media_meta['image'])
+
+            thumbnails = [{
+                'url': image.get('href'),
+                'width': int_or_none(image.get('width')),
+                'height': int_or_none(image.get('height')),
+            } for image in images]
+
+            entries.append({
+                'id': video_id,
+                'title': title,
+                'thumbnails': thumbnails,
+                'duration': duration,
+                'timestamp': timestamp,
+                'formats': formats,
+                'subtitles': subtitles,
+            })
+
+        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)

youtube_dl/extractor/comcarcoff.py

@@ -36,7 +36,7 @@ class ComCarCoffIE(InfoExtractor):
             webpage, 'full data json'))
 
         video_id = full_data['activeVideo']['video']
-        video_data = full_data['videos'][video_id]
+        video_data = full_data.get('videos', {}).get(video_id) or full_data['singleshots'][video_id]
         thumbnails = [{
             'url': video_data['images']['thumb'],
         }, {

youtube_dl/extractor/ir90tv.py

@@ -0,0 +1,42 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import remove_start
+
+
+class Ir90TvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?90tv\.ir/video/(?P<id>[0-9]+)/.*'
+    _TESTS = [{
+        'url': 'http://90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
+        'md5': '411dbd94891381960cb9e13daa47a869',
+        'info_dict': {
+            'id': '95719',
+            'ext': 'mp4',
+            'title': 'شایعات نقل و انتقالات مهم فوتبال اروپا 94/02/18',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = remove_start(self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title'), '90tv.ir :: ')
+
+        video_url = self._search_regex(
+            r'<source[^>]+src="([^"]+)"', webpage, 'video url')
+
+        thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url', fatal=False)
+
+        return {
+            'url': video_url,
+            'id': video_id,
+            'title': title,
+            'video_url': video_url,
+            'thumbnail': thumbnail,
+        }

youtube_dl/extractor/soundcloud.py

@@ -282,9 +282,11 @@ class SoundcloudSetIE(SoundcloudIE):
             msgs = (compat_str(err['error_message']) for err in info['errors'])
             raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
 
+        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in info['tracks']]
+
         return {
             '_type': 'playlist',
-            'entries': [self._extract_info_dict(track, secret_token=token) for track in info['tracks']],
+            'entries': entries,
             'id': '%s' % info['id'],
             'title': info['title'],
         }
@@ -379,9 +381,7 @@ class SoundcloudPlaylistIE(SoundcloudIE):
         data = self._download_json(
             base_url + data, playlist_id, 'Downloading playlist')
 
-        entries = [
-            self._extract_info_dict(t, quiet=True, secret_token=token)
-            for t in data['tracks']]
+        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in data['tracks']]
 
         return {
             '_type': 'playlist',

youtube_dl/extractor/spiegel.py

@@ -9,7 +9,7 @@ from .spiegeltv import SpiegeltvIE
 
 
 class SpiegelIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed)?(?:\.html)?(?:#.*)?$'
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed|-iframe)?(?:\.html)?(?:#.*)?$'
     _TESTS = [{
         'url': 'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
         'md5': '2c2754212136f35fb4b19767d242f66e',
@@ -39,6 +39,9 @@ class SpiegelIE(InfoExtractor):
             'description': 'SPIEGEL ONLINE-Nutzer durften den deutschen Astronauten Alexander Gerst über sein Leben auf der ISS-Station befragen. Hier kommen seine Antworten auf die besten sechs Fragen.',
             'title': 'Fragen an Astronaut Alexander Gerst: "Bekommen Sie die Tageszeiten mit?"',
         }
+    }, {
+        'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-iframe.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

youtube_dl/extractor/youtube.py

@@ -281,13 +281,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
-        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'VP9'},
+        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'vp9'},
         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
@@ -297,11 +297,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'VP9'},
-        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'vp9'},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
 
         # Dash webm audio
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
@@ -993,7 +993,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if 'reason' in video_info:
                 if 'The uploader has not made this video available in your country.' in video_info['reason']:
                     regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
-                    if regions_allowed is not None:
+                    if regions_allowed:
                         raise ExtractorError('YouTube said: This video is available in %s only' % (
                             ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
                             expected=True)

youtube_dl/options.py

@@ -184,7 +184,8 @@ def parseOpts(overrideArguments=None):
     network.add_option(
         '--socks-proxy', dest='socksproxy', default=None, metavar='URL',
         help=('Use the specified socks proxy. Pass in an empty string '
-              '(--socks-proxy "") for direct connection'))
+              '(--socks-proxy "") for direct connection. This feature requires'
+              'the pysocks library.'))
     network.add_option(
         '--socket-timeout',
         dest='socket_timeout', type=float, default=None, metavar='SECONDS',

youtube_dl/utils.py

@@ -585,6 +585,23 @@ class ContentTooShortError(Exception):
         self.expected = expected
 
 
+class OptionalDependencyNotFound(Exception):
+    """Optional dependency not found
+
+    This exception may be raised by YoutubeDL, when the user tries to use a
+    feature that requires an optional dependency which could not be found.
+    """
+
+    def __init__(self, module_name, feature_name):
+        self.module_name = module_name
+        self.feature_name = feature_name
+
+    def __str__(self):
+        return ("Unable to use {feature}, because it depends on {module} "
+                "which was not found.").format(
+            feature=self.feature_name, module=self.module_name)
+
+
 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     hc = http_class(*args, **kwargs)
     source_address = ydl_handler._params.get('source_address')