From a9308ca6015cd53e6ed35098daaf9c82b90f5554 Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 5 Feb 2016 09:39:13 +0100 Subject: [PATCH 1/2] Add --skip-protocols to exclude specific protocols from format extraction --- youtube_dl/__init__.py | 1 + youtube_dl/extractor/common.py | 8 ++++++++ youtube_dl/options.py | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index f5f064241..2dfa40082 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -373,6 +373,7 @@ def _real_main(argv=None): 'external_downloader_args': external_downloader_args, 'postprocessor_args': postprocessor_args, 'cn_verification_proxy': opts.cn_verification_proxy, + 'skip_protocols': opts.skip_protocols, } with YoutubeDL(ydl_opts) as ydl: diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 199a04d1c..2305adcd9 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -939,6 +939,8 @@ class InfoExtractor(object): def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None, transform_source=lambda s: fix_xml_ampersands(s).strip(), fatal=True): + if 'f4m' in self._downloader.params.get('skip_protocols'): + return [] manifest = self._download_xml( manifest_url, video_id, 'Downloading f4m manifest', 'Unable to download f4m manifest', @@ -995,6 +997,8 @@ class InfoExtractor(object): entry_protocol='m3u8', preference=None, m3u8_id=None, note=None, errnote=None, fatal=True): + if 'm3u8' in self._downloader.params.get('skip_protocols'): + return [] formats = [{ 'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])), @@ -1105,6 +1109,8 @@ class InfoExtractor(object): return '/'.join(out) def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None): + if 'smil' in self._downloader.params.get('skip_protocols'): + return [] smil = self._download_smil(smil_url, video_id, fatal=fatal) if smil is False: @@ -1338,6 +1344,8 @@ class InfoExtractor(object): fatal=fatal) def _extract_dash_manifest_formats(self, dash_manifest_url, video_id, fatal=True, namespace=None, formats_dict={}): + if 'dash' in self._downloader.params.get('skip_protocols'): + return [] dash_doc = self._download_dash_manifest(dash_manifest_url, video_id, fatal) if dash_doc is False: return [] diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 2137dfb3f..b953a55c4 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -354,6 +354,11 @@ def parseOpts(overrideArguments=None): 'If a merge is required (e.g. bestvideo+bestaudio), ' 'output to given container format. One of mkv, mp4, ogg, webm, flv. ' 'Ignored if no merge is required')) + video_format.add_option( + '--skip-protocols', + action='callback', dest='skip_protocols', type='str', + default=[], callback=_comma_separated_values_options_callback, + help='Protocols to exclude from the format extraction separated by commas (Possible values: m3u8, f4m, dash and smil)') subtitles = optparse.OptionGroup(parser, 'Subtitle Options') subtitles.add_option( From 2500db0de78bac46d054e4d9e288a92467461049 Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 5 Feb 2016 12:34:28 +0100 Subject: [PATCH 2/2] replace --youtube-skip-dash-manifest with --skip-protocols dash --- youtube_dl/__init__.py | 1 - youtube_dl/extractor/common.py | 8 ++++---- youtube_dl/extractor/youtube.py | 12 +++++------- youtube_dl/options.py | 8 -------- 4 files changed, 9 insertions(+), 20 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 2dfa40082..76dd60011 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -352,7 +352,6 @@ def _real_main(argv=None): 'prefer_ffmpeg': opts.prefer_ffmpeg, 'include_ads': opts.include_ads, 'default_search': opts.default_search, - 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest, 'encoding': opts.encoding, 'extract_flat': opts.extract_flat, 'merge_output_format': opts.merge_output_format, diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 2305adcd9..98fbd9abf 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -939,7 +939,7 @@ class InfoExtractor(object): def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None, transform_source=lambda s: fix_xml_ampersands(s).strip(), fatal=True): - if 'f4m' in self._downloader.params.get('skip_protocols'): + if 'f4m' in self._downloader.params.get('skip_protocols', []): return [] manifest = self._download_xml( manifest_url, video_id, 'Downloading f4m manifest', @@ -997,7 +997,7 @@ class InfoExtractor(object): entry_protocol='m3u8', preference=None, m3u8_id=None, note=None, errnote=None, fatal=True): - if 'm3u8' in self._downloader.params.get('skip_protocols'): + if 'm3u8' in self._downloader.params.get('skip_protocols', []): return [] formats = [{ @@ -1109,7 +1109,7 @@ class InfoExtractor(object): return '/'.join(out) def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None): - if 'smil' in self._downloader.params.get('skip_protocols'): + if 'smil' in self._downloader.params.get('skip_protocols', []): return [] smil = self._download_smil(smil_url, video_id, fatal=fatal) @@ -1344,7 +1344,7 @@ class InfoExtractor(object): fatal=fatal) def _extract_dash_manifest_formats(self, dash_manifest_url, video_id, fatal=True, namespace=None, formats_dict={}): - if 'dash' in self._downloader.params.get('skip_protocols'): + if 'dash' in self._downloader.params.get('skip_protocols', []): return [] dash_doc = self._download_dash_manifest(dash_manifest_url, video_id, fatal) if dash_doc is False: diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index e9ed77e42..0427fd60e 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -471,7 +471,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'title': 'UHDTV TEST 8K VIDEO.mp4' }, 'params': { - 'youtube_include_dash_manifest': True, 'format': '141', }, }, @@ -488,7 +487,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'upload_date': '20131011', }, 'params': { - 'youtube_include_dash_manifest': True, 'format': '141', }, }, @@ -507,7 +505,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'creator': 'Taylor Swift', }, 'params': { - 'youtube_include_dash_manifest': True, 'format': '141', }, }, @@ -645,9 +642,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015', }, 'params': { - 'youtube_include_dash_manifest': True, 'format': '135', # bestvideo - } + }, + 'skip': 'This live event has ended.', }, { # Multifeed videos (multiple cameras), URL is for Main Camera @@ -747,6 +744,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'params': { 'skip_download': True, }, + 'skip': 'YouTube said: This video does not exist.', }, { 'url': 'https://www.youtube.com/watch?feature=player_embedded&v=V36LpHqtcDY', @@ -1113,7 +1111,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): add_dash_mpd(video_info) if args.get('livestream') == '1' or args.get('live_playback') == 1: is_live = True - if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True): + if not video_info or 'dash' not in self._downloader.params.get('skip_protocols', []): # We also try looking in get_video_info since it may contain different dashmpd # URL that points to a DASH manifest with possibly different itag set (some itags # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH @@ -1461,7 +1459,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info') # Look for the DASH manifest - if self._downloader.params.get('youtube_include_dash_manifest', True): + if 'dash' not in self._downloader.params.get('skip_protocols', []): dash_mpd_fatal = True for dash_manifest_url in dash_mpds: dash_formats = {} diff --git a/youtube_dl/options.py b/youtube_dl/options.py index b953a55c4..cabcfdfe7 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -339,14 +339,6 @@ def parseOpts(overrideArguments=None): '-F', '--list-formats', action='store_true', dest='listformats', help='List all available formats of requested videos') - video_format.add_option( - '--youtube-include-dash-manifest', - action='store_true', dest='youtube_include_dash_manifest', default=True, - help=optparse.SUPPRESS_HELP) - video_format.add_option( - '--youtube-skip-dash-manifest', - action='store_false', dest='youtube_include_dash_manifest', - help='Do not download the DASH manifests and related data on YouTube videos') video_format.add_option( '--merge-output-format', action='store', dest='merge_output_format', metavar='FORMAT', default=None,