From 75f86b2a66023d2901259ae85fe7435532d6c410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 17 Mar 2016 14:15:49 +0100 Subject: [PATCH 01/11] extractors: don't access directly params from the downloader --- youtube_dl/extractor/brightcove.py | 2 +- youtube_dl/extractor/ccc.py | 2 +- youtube_dl/extractor/common.py | 29 +++++++++++++------------- youtube_dl/extractor/commonmistakes.py | 2 +- youtube_dl/extractor/daum.py | 2 +- youtube_dl/extractor/deezer.py | 2 +- youtube_dl/extractor/generic.py | 6 +++--- youtube_dl/extractor/leeco.py | 2 +- youtube_dl/extractor/nba.py | 2 +- youtube_dl/extractor/neteasemusic.py | 2 +- youtube_dl/extractor/pluralsight.py | 6 +++--- youtube_dl/extractor/smotri.py | 4 ++-- youtube_dl/extractor/sohu.py | 2 +- youtube_dl/extractor/vimeo.py | 6 +++--- youtube_dl/extractor/youku.py | 4 ++-- youtube_dl/extractor/youtube.py | 18 ++++++++-------- 16 files changed, 46 insertions(+), 45 deletions(-) diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index 0d162d337..70d44cb63 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -343,7 +343,7 @@ class BrightcoveLegacyIE(InfoExtractor): 'url': video_info['FLVFullLengthURL'], }) - if self._downloader.params.get('include_ads', False): + if self.params.get('include_ads', False): adServerURL = video_info.get('_youtubedl_adServerURL') if adServerURL: ad_info = { diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py index dda2c0959..9341876ac 100644 --- a/youtube_dl/extractor/ccc.py +++ b/youtube_dl/extractor/ccc.py @@ -37,7 +37,7 @@ class CCCIE(InfoExtractor): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - if self._downloader.params.get('prefer_free_formats'): + if self.params.get('prefer_free_formats'): preference = qualities(['mp3', 'opus', 'mp4-lq', 'webm-lq', 'h264-sd', 'mp4-sd', 'webm-sd', 'mp4', 'webm', 'mp4-hd', 'h264-hd', 'webm-hd']) else: preference = qualities(['opus', 'mp3', 'webm-lq', 'mp4-lq', 'webm-sd', 'h264-sd', 'mp4-sd', 'webm', 'mp4', 'webm-hd', 'mp4-hd', 'h264-hd']) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 85ac0400c..d0bb8d4fe 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -331,6 +331,7 @@ class InfoExtractor(object): def set_downloader(self, downloader): """Sets the downloader for this IE.""" self._downloader = downloader + self.params = downloader.params if downloader else {} def _real_initialize(self): """Real initialization process. Redefine in subclasses.""" @@ -419,7 +420,7 @@ class InfoExtractor(object): webpage_bytes = prefix + webpage_bytes if not encoding: encoding = self._guess_encoding_from_content(content_type, webpage_bytes) - if self._downloader.params.get('dump_intermediate_pages', False): + if self.params.get('dump_intermediate_pages', False): try: url = url_or_request.get_full_url() except AttributeError: @@ -427,7 +428,7 @@ class InfoExtractor(object): self.to_screen('Dumping request to ' + url) dump = base64.b64encode(webpage_bytes).decode('ascii') self._downloader.to_screen(dump) - if self._downloader.params.get('write_pages', False): + if self.params.get('write_pages', False): try: url = url_or_request.get_full_url() except AttributeError: @@ -610,7 +611,7 @@ class InfoExtractor(object): if mobj: break - if not self._downloader.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty(): + if not self.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty(): _name = '\033[0;34m%s\033[0m' % name else: _name = name @@ -650,7 +651,7 @@ class InfoExtractor(object): username = None password = None - downloader_params = self._downloader.params + downloader_params = self.params # Attempt to use provided username and password or .netrc data if downloader_params.get('username') is not None: @@ -678,7 +679,7 @@ class InfoExtractor(object): """ if self._downloader is None: return None - downloader_params = self._downloader.params + downloader_params = self.params if downloader_params.get('twofactor') is not None: return downloader_params['twofactor'] @@ -869,7 +870,7 @@ class InfoExtractor(object): if f.get('vcodec') == 'none': # audio only preference -= 50 - if self._downloader.params.get('prefer_free_formats'): + if self.params.get('prefer_free_formats'): ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus'] else: ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a'] @@ -881,7 +882,7 @@ class InfoExtractor(object): else: if f.get('acodec') == 'none': # video only preference -= 40 - if self._downloader.params.get('prefer_free_formats'): + if self.params.get('prefer_free_formats'): ORDER = ['flv', 'mp4', 'webm'] else: ORDER = ['webm', 'flv', 'mp4'] @@ -948,7 +949,7 @@ class InfoExtractor(object): """ Either "http:" or "https:", depending on the user's preferences """ return ( 'http:' - if self._downloader.params.get('prefer_insecure', False) + if self.params.get('prefer_insecure', False) else 'https:') def _proto_relative_url(self, url, scheme=None): @@ -1614,8 +1615,8 @@ class InfoExtractor(object): return not any_restricted def extract_subtitles(self, *args, **kwargs): - if (self._downloader.params.get('writesubtitles', False) or - self._downloader.params.get('listsubtitles')): + if (self.params.get('writesubtitles', False) or + self.params.get('listsubtitles')): return self._get_subtitles(*args, **kwargs) return {} @@ -1640,8 +1641,8 @@ class InfoExtractor(object): return ret def extract_automatic_captions(self, *args, **kwargs): - if (self._downloader.params.get('writeautomaticsub', False) or - self._downloader.params.get('listsubtitles')): + if (self.params.get('writeautomaticsub', False) or + self.params.get('listsubtitles')): return self._get_automatic_captions(*args, **kwargs) return {} @@ -1649,9 +1650,9 @@ class InfoExtractor(object): raise NotImplementedError('This method must be implemented by subclasses') def mark_watched(self, *args, **kwargs): - if (self._downloader.params.get('mark_watched', False) and + if (self.params.get('mark_watched', False) and (self._get_login_info()[0] is not None or - self._downloader.params.get('cookiefile') is not None)): + self.params.get('cookiefile') is not None)): self._mark_watched(*args, **kwargs) def _mark_watched(self, *args, **kwargs): diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py index 2f86e2381..998e340f5 100644 --- a/youtube_dl/extractor/commonmistakes.py +++ b/youtube_dl/extractor/commonmistakes.py @@ -24,7 +24,7 @@ class CommonMistakesIE(InfoExtractor): 'That doesn\'t make any sense. ' 'Simply remove the parameter in your command or configuration.' ) % url - if not self._downloader.params.get('verbose'): + if not self.params.get('verbose'): msg += ' Add -v to the command line to see what arguments and configuration youtube-dl got.' raise ExtractorError(msg, expected=True) diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py index 86024a745..6a6b98f14 100644 --- a/youtube_dl/extractor/daum.py +++ b/youtube_dl/extractor/daum.py @@ -190,7 +190,7 @@ class DaumListIE(InfoExtractor): query_dict = compat_parse_qs(compat_urlparse.urlparse(url).query) if 'clipid' in query_dict: clip_id = query_dict['clipid'][0] - if self._downloader.params.get('noplaylist'): + if self.params.get('noplaylist'): self.to_screen('Downloading just video %s because of --no-playlist' % clip_id) return self.url_result(DaumClipIE._URL_TEMPLATE % clip_id, 'DaumClip') else: diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index c3205ff5f..9a151b587 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -26,7 +26,7 @@ class DeezerPlaylistIE(InfoExtractor): } def _real_extract(self, url): - if 'test' not in self._downloader.params: + if 'test' not in self.params: self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') mobj = re.match(self._VALID_URL, url) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 589d1e152..605df716e 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -1212,7 +1212,7 @@ class GenericIE(InfoExtractor): parsed_url = compat_urlparse.urlparse(url) if not parsed_url.scheme: - default_search = self._downloader.params.get('default_search') + default_search = self.params.get('default_search') if default_search is None: default_search = 'fixup_error' @@ -1301,8 +1301,8 @@ class GenericIE(InfoExtractor): info_dict['formats'] = formats return info_dict - if not self._downloader.params.get('test', False) and not is_intentional: - force = self._downloader.params.get('force_generic_extractor', False) + if not self.params.get('test', False) and not is_intentional: + force = self.params.get('force_generic_extractor', False) self._downloader.report_warning( '%s on generic information extractor.' % ('Forcing' if force else 'Falling back')) diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py index 375fdaed1..58df591fb 100644 --- a/youtube_dl/extractor/leeco.py +++ b/youtube_dl/extractor/leeco.py @@ -124,7 +124,7 @@ class LeIE(InfoExtractor): play_json_req = sanitized_Request( 'http://api.le.com/mms/out/video/playJson?' + compat_urllib_parse_urlencode(params) ) - cn_verification_proxy = self._downloader.params.get('cn_verification_proxy') + cn_verification_proxy = self.params.get('cn_verification_proxy') if cn_verification_proxy: play_json_req.add_header('Ytdl-request-proxy', cn_verification_proxy) diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py index d896b0d04..e6ae1e4d8 100644 --- a/youtube_dl/extractor/nba.py +++ b/youtube_dl/extractor/nba.py @@ -113,7 +113,7 @@ class NBAIE(InfoExtractor): def _extract_playlist(self, orig_path, video_id, webpage): team = orig_path.split('/')[0] - if self._downloader.params.get('noplaylist'): + if self.params.get('noplaylist'): self.to_screen('Downloading just video because of --no-playlist') video_path = self._search_regex( r'nbaVideoCore\.firstVideo\s*=\s*\'([^\']+)\';', webpage, 'video path') diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py index 0d36474fa..a3adf97a7 100644 --- a/youtube_dl/extractor/neteasemusic.py +++ b/youtube_dl/extractor/neteasemusic.py @@ -392,7 +392,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE): name = info['name'] description = info['description'] - if not info['songs'] or self._downloader.params.get('noplaylist'): + if not info['songs'] or self.params.get('noplaylist'): if info['songs']: self.to_screen( 'Downloading just the main audio %s because of --no-playlist' diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py index df03dd419..9e32ed10b 100644 --- a/youtube_dl/extractor/pluralsight.py +++ b/youtube_dl/extractor/pluralsight.py @@ -167,18 +167,18 @@ class PluralsightIE(PluralsightBaseIE): # In order to minimize the number of calls to ViewClip API and reduce # the probability of being throttled or banned by Pluralsight we will request # only single format until formats listing was explicitly requested. - if self._downloader.params.get('listformats', False): + if self.params.get('listformats', False): allowed_qualities = ALLOWED_QUALITIES else: def guess_allowed_qualities(): - req_format = self._downloader.params.get('format') or 'best' + req_format = self.params.get('format') or 'best' req_format_split = req_format.split('-', 1) if len(req_format_split) > 1: req_ext, req_quality = req_format_split for allowed_quality in ALLOWED_QUALITIES: if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities: return (AllowedQuality(req_ext, (req_quality, )), ) - req_ext = 'webm' if self._downloader.params.get('prefer_free_formats') else 'mp4' + req_ext = 'webm' if self.params.get('prefer_free_formats') else 'mp4' return (AllowedQuality(req_ext, (best_quality, )), ) allowed_qualities = guess_allowed_qualities() diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py index 5c3fd0fec..d7c6b3266 100644 --- a/youtube_dl/extractor/smotri.py +++ b/youtube_dl/extractor/smotri.py @@ -170,7 +170,7 @@ class SmotriIE(InfoExtractor): 'getvideoinfo': '1', } - video_password = self._downloader.params.get('videopassword') + video_password = self.params.get('videopassword') if video_password: video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest() @@ -356,7 +356,7 @@ class SmotriBroadcastIE(InfoExtractor): url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket - broadcast_password = self._downloader.params.get('videopassword') + broadcast_password = self.params.get('videopassword') if broadcast_password: url += '&pass=%s' % hashlib.md5(broadcast_password.encode('utf-8')).hexdigest() diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py index 49e5d09ae..f587503aa 100644 --- a/youtube_dl/extractor/sohu.py +++ b/youtube_dl/extractor/sohu.py @@ -98,7 +98,7 @@ class SohuIE(InfoExtractor): req = sanitized_Request(base_data_url + vid_id) - cn_verification_proxy = self._downloader.params.get('cn_verification_proxy') + cn_verification_proxy = self.params.get('cn_verification_proxy') if cn_verification_proxy: req.add_header('Ytdl-request-proxy', cn_verification_proxy) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 707a5735a..fbce4e3db 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -250,7 +250,7 @@ class VimeoIE(VimeoBaseInfoExtractor): return mobj.group(1) def _verify_video_password(self, url, video_id, webpage): - password = self._downloader.params.get('videopassword') + password = self.params.get('videopassword') if password is None: raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True) token, vuid = self._extract_xsrft_and_vuid(webpage) @@ -270,7 +270,7 @@ class VimeoIE(VimeoBaseInfoExtractor): 'Verifying the password', 'Wrong password') def _verify_player_video_password(self, url, video_id): - password = self._downloader.params.get('videopassword') + password = self.params.get('videopassword') if password is None: raise ExtractorError('This video is protected by a password, use the --video-password option') data = urlencode_postdata({'password': password}) @@ -567,7 +567,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor): if not login_form: return webpage - password = self._downloader.params.get('videopassword') + password = self.params.get('videopassword') if password is None: raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True) fields = self._hidden_inputs(login_form) diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py index fd7eb5a6d..8a5b5fb3a 100644 --- a/youtube_dl/extractor/youku.py +++ b/youtube_dl/extractor/youku.py @@ -206,7 +206,7 @@ class YoukuIE(InfoExtractor): self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com') req = sanitized_Request(req_url, headers=headers) - cn_verification_proxy = self._downloader.params.get('cn_verification_proxy') + cn_verification_proxy = self.params.get('cn_verification_proxy') if cn_verification_proxy: req.add_header('Ytdl-request-proxy', cn_verification_proxy) @@ -214,7 +214,7 @@ class YoukuIE(InfoExtractor): return raw_data['data'] - video_password = self._downloader.params.get('videopassword') + video_password = self.params.get('videopassword') # request basic data basic_data_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % video_id diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 28355bf46..28f00d08f 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -888,7 +888,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): download_note = ( 'Downloading player %s' % player_url - if self._downloader.params.get('verbose') else + if self.params.get('verbose') else 'Downloading %s player %s' % (player_type, player_id) ) if player_type == 'js': @@ -985,7 +985,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): ) self._player_cache[player_id] = func func = self._player_cache[player_id] - if self._downloader.params.get('youtube_print_sig_code'): + if self.params.get('youtube_print_sig_code'): self._print_sig_code(func, s) return func(s) except Exception as e: @@ -1179,7 +1179,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): url, smuggled_data = unsmuggle_url(url, {}) proto = ( - 'http' if self._downloader.params.get('prefer_insecure', False) + 'http' if self.params.get('prefer_insecure', False) else 'https') start_time = None @@ -1253,7 +1253,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): add_dash_mpd(video_info) if args.get('livestream') == '1' or args.get('live_playback') == 1: is_live = True - if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True): + if not video_info or self.params.get('youtube_include_dash_manifest', True): # We also try looking in get_video_info since it may contain different dashmpd # URL that points to a DASH manifest with possibly different itag set (some itags # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH @@ -1331,7 +1331,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): video_description = '' if 'multifeed_metadata_list' in video_info and not smuggled_data.get('force_singlefeed', False): - if not self._downloader.params.get('noplaylist'): + if not self.params.get('noplaylist'): entries = [] feed_ids = [] multifeed_metadata_list = video_info['multifeed_metadata_list'][0] @@ -1457,7 +1457,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # annotations video_annotations = None - if self._downloader.params.get('writeannotations', False): + if self.params.get('writeannotations', False): video_annotations = self._extract_annotations(video_id) def _map_to_format_list(urlmap): @@ -1532,7 +1532,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): video_webpage, 'age gate player URL') player_url = json.loads(player_url_json) - if self._downloader.params.get('verbose'): + if self.params.get('verbose'): if player_url is None: player_version = 'unknown' player_desc = 'unknown' @@ -1627,7 +1627,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info') # Look for the DASH manifest - if self._downloader.params.get('youtube_include_dash_manifest', True): + if self.params.get('youtube_include_dash_manifest', True): dash_mpd_fatal = True for mpd_url in dash_mpds: dash_formats = {} @@ -1862,7 +1862,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor): query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query) if 'v' in query_dict: video_id = query_dict['v'][0] - if self._downloader.params.get('noplaylist'): + if self.params.get('noplaylist'): self.to_screen('Downloading just video %s because of --no-playlist' % video_id) return self.url_result(video_id, 'Youtube', video_id=video_id) else: From b67e4daddbf14a539d23a21415f24b01281d4c69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 18 Mar 2016 19:02:09 +0100 Subject: [PATCH 02/11] Allow passing different options to each extractor Currently it can only be done through python and only affects the extractors, not YoutubeDL. --- test/test_YoutubeDL.py | 17 ++++++++++++++ youtube_dl/YoutubeDL.py | 12 +++++++--- youtube_dl/extractor/common.py | 3 ++- youtube_dl/params.py | 43 ++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 youtube_dl/params.py diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index ca25025e2..d35fb53a0 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -17,6 +17,7 @@ from youtube_dl.extractor import YoutubeIE from youtube_dl.extractor.common import InfoExtractor from youtube_dl.postprocessor.common import PostProcessor from youtube_dl.utils import ExtractorError, match_filter_func +from youtube_dl.params import Params TEST_URL = 'http://localhost/sample.mp4' @@ -702,6 +703,22 @@ class TestYoutubeDL(unittest.TestCase): downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['url'], TEST_URL) + def test_subparams(self): + example_params = {'foo': 'example'} + params = Params({'foo': 'base', 'blah': 'base'}, {'example.com': example_params}) + ydl = YoutubeDL(params) + + class ExampleIE(InfoExtractor): + IE_NAME = 'example.com' + + ie = ExampleIE() + ydl.add_info_extractor(ie) + pars = ie.params + self.assertEqual(pars['foo'], 'example') + self.assertEqual(pars['blah'], 'base') + self.assertEqual(pars.get('blah'), 'base') + self.assertEqual(pars.get('nonexistant'), None) + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index d7aa951ff..0c644a3ab 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -94,6 +94,7 @@ from .postprocessor import ( get_postprocessor, ) from .version import __version__ +from .params import Params if compat_os_name == 'nt': import ctypes @@ -122,7 +123,9 @@ class YoutubeDL(object): options instead. These options are available through the params attribute for the InfoExtractors to use. The YoutubeDL also registers itself as the downloader in charge for the InfoExtractors - that are added to it, so this is a "mutual registration". + that are added to it, so this is a "mutual registration". To use different + parameters in an extractor, the youtube_dl.params.Params class must be + used. Available options: @@ -294,11 +297,14 @@ class YoutubeDL(object): self._num_downloads = 0 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self._err_file = sys.stderr - self.params = { + if not isinstance(params, Params): + params = Params(params) + self.params = params + default_params = { # Default parameters 'nocheckcertificate': False, } - self.params.update(params) + self.add_extra_info(self.params, default_params) self.cache = Cache(self) if params.get('bidi_workaround', False): diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index d0bb8d4fe..b932e9eb2 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -53,6 +53,7 @@ from ..utils import ( update_Request, update_url_query, ) +from ..params import ParamsSection class InfoExtractor(object): @@ -331,7 +332,7 @@ class InfoExtractor(object): def set_downloader(self, downloader): """Sets the downloader for this IE.""" self._downloader = downloader - self.params = downloader.params if downloader else {} + self.params = downloader.params.section(self.IE_NAME) if downloader else ParamsSection() def _real_initialize(self): """Real initialization process. Redefine in subclasses.""" diff --git a/youtube_dl/params.py b/youtube_dl/params.py new file mode 100644 index 000000000..03d2f2f2e --- /dev/null +++ b/youtube_dl/params.py @@ -0,0 +1,43 @@ +from __future__ import unicode_literals + + +class Params(dict): + """Params class + + The params class holds the parameters for YoutubeDL objects, in its + simplest form it's initialized with a dictionary with the parameters. To + override some parameter in an info extractor a dictionary can be passed as + the second argument, its keys must match the IE_NAME properties of the + extractors. + """ + def __init__(self, params, sections=None): + super(Params, self).__init__(params) + if sections is None: + sections = {} + self.sections = sections + + def section(self, section): + """Return the params for the specified section""" + return ParamsSection(self.sections.get(section, {}), self) + + +class ParamsSection(object): + def __init__(self, main=None, parent=None): + if main is None: + main = {} + if parent is None: + parent = Params({}) + self.main = main + self.parent = parent + + def __getitem__(self, key): + if key in self.main: + return self.main[key] + else: + return self.parent[key] + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default From 09495f1d120e5e75e21a684642af6755480fc0a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 18 Mar 2016 22:30:21 +0100 Subject: [PATCH 03/11] YoutubeDL: use parameters of the specific extractor The 'force*' and 'extract_flat' parameters are only looked in the main parameters dict, because I think that it only makes sense to use them in a global way --- test/test_YoutubeDL.py | 37 +++++++++++---- test/test_download.py | 4 +- youtube_dl/YoutubeDL.py | 101 ++++++++++++++++++++++------------------ 3 files changed, 87 insertions(+), 55 deletions(-) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index d35fb53a0..86be58dcb 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -28,7 +28,7 @@ class YDL(FakeYDL): self.downloaded_info_dicts = [] self.msgs = [] - def process_info(self, info_dict): + def process_info(self, info_dict, params): self.downloaded_info_dicts.append(info_dict) def to_screen(self, msg): @@ -438,7 +438,7 @@ class TestYoutubeDL(unittest.TestCase): params.setdefault('simulate', True) ydl = YDL(params) ydl.report_warning = lambda *args, **kargs: None - return ydl.process_video_result(info_dict, download=False) + return ydl.process_video_result(info_dict, params, download=False) result = get_info() self.assertFalse(result.get('requested_subtitles')) @@ -527,7 +527,7 @@ class TestYoutubeDL(unittest.TestCase): f.write('EXAMPLE') ydl = YoutubeDL(params) ydl.add_post_processor(PP()) - ydl.post_process(filename, {'filepath': filename}) + ydl.post_process(filename, {'filepath': filename}, params) run_pp({'keepvideo': True}, SimplePP) self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename) @@ -556,8 +556,8 @@ class TestYoutubeDL(unittest.TestCase): super(FilterYDL, self).__init__(*args, **kwargs) self.params['simulate'] = True - def process_info(self, info_dict): - super(YDL, self).process_info(info_dict) + def process_info(self, info_dict, params): + super(YDL, self).process_info(info_dict, params) def _match_entry(self, info_dict, incomplete): res = super(FilterYDL, self)._match_entry(info_dict, incomplete) @@ -704,12 +704,30 @@ class TestYoutubeDL(unittest.TestCase): self.assertEqual(downloaded['url'], TEST_URL) def test_subparams(self): - example_params = {'foo': 'example'} - params = Params({'foo': 'base', 'blah': 'base'}, {'example.com': example_params}) - ydl = YoutubeDL(params) + example_params = {'foo': 'example', 'outtmpl': 'foo.mp4'} + params = Params( + {'foo': 'base', 'blah': 'base', 'skip_download': True}, {'example.com': example_params}) + ydl = YoutubeDL(params, auto_init=False) + ydl.downloads = [] + real_process_info = ydl.process_info + def process_info(info_dict, params): + r = real_process_info(info_dict, params) + ydl.downloads.append(info_dict) + return r + ydl.process_info = process_info class ExampleIE(InfoExtractor): IE_NAME = 'example.com' + _VALID_URL = r'example$' + + def _real_extract(self, url): + return { + 'id': '1', + 'ext': 'mp4', + 'title': 'example', + 'url': 'http://example.com', + } + ie = ExampleIE() ydl.add_info_extractor(ie) @@ -719,6 +737,9 @@ class TestYoutubeDL(unittest.TestCase): self.assertEqual(pars.get('blah'), 'base') self.assertEqual(pars.get('nonexistant'), None) + ydl.extract_info('example') + self.assertEqual(ydl.downloads[-1]['_filename'], 'foo.mp4') + if __name__ == '__main__': unittest.main() diff --git a/test/test_download.py b/test/test_download.py index a3f1c0644..2740b755f 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -51,9 +51,9 @@ class YoutubeDL(youtube_dl.YoutubeDL): # Don't accept warnings during tests raise ExtractorError(message) - def process_info(self, info_dict): + def process_info(self, info_dict, params): self.processed_info_dicts.append(info_dict) - return super(YoutubeDL, self).process_info(info_dict) + return super(YoutubeDL, self).process_info(info_dict, params) def _file_md5(fn): diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 0c644a3ab..e5e7482f6 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -556,13 +556,15 @@ class YoutubeDL(object): except UnicodeEncodeError: self.to_screen('[download] The file has already been downloaded') - def prepare_filename(self, info_dict): + def prepare_filename(self, info_dict, params=None): """Generate the output filename.""" + if params is None: + params = self.params try: template_dict = dict(info_dict) template_dict['epoch'] = int(time.time()) - autonumber_size = self.params.get('autonumber_size') + autonumber_size = params.get('autonumber_size') if autonumber_size is None: autonumber_size = 5 autonumber_templ = '%0' + str(autonumber_size) + 'd' @@ -579,14 +581,14 @@ class YoutubeDL(object): sanitize = lambda k, v: sanitize_filename( compat_str(v), - restricted=self.params.get('restrictfilenames'), + restricted=params.get('restrictfilenames'), is_id=(k == 'id')) template_dict = dict((k, sanitize(k, v)) for k, v in template_dict.items() if v is not None) template_dict = collections.defaultdict(lambda: 'NA', template_dict) - outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL) + outtmpl = params.get('outtmpl', DEFAULT_OUTTMPL) tmpl = compat_expanduser(outtmpl) filename = tmpl % template_dict # Temporary fix for #4787 @@ -683,7 +685,9 @@ class YoutubeDL(object): } self.add_default_extra_info(ie_result, ie, url) if process: - return self.process_ie_result(ie_result, download, extra_info) + return self.process_ie_result( + ie_result, download, extra_info, + params=self.params.section(ie.IE_NAME)) else: return ie_result except ExtractorError as e: # An error we somewhat expected @@ -708,7 +712,7 @@ class YoutubeDL(object): 'extractor_key': ie.ie_key(), }) - def process_ie_result(self, ie_result, download=True, extra_info={}): + def process_ie_result(self, ie_result, download=True, extra_info={}, params=None): """ Take the result of the ie(may be modified) and resolve all unresolved references (URLs, playlist items). @@ -716,6 +720,8 @@ class YoutubeDL(object): It will also download the videos if 'download'. Returns the resolved ie_result. """ + if params is None: + params = self.params result_type = ie_result.get('_type', 'video') if result_type in ('url', 'url_transparent'): @@ -728,7 +734,7 @@ class YoutubeDL(object): if result_type == 'video': self.add_extra_info(ie_result, extra_info) - return self.process_video_result(ie_result, download=download) + return self.process_video_result(ie_result, params, download=download) elif result_type == 'url': # We have to add extra_info to the results because it may be # contained in a playlist @@ -753,7 +759,7 @@ class YoutubeDL(object): assert new_result.get('_type') != 'url_transparent' return self.process_ie_result( - new_result, download=download, extra_info=extra_info) + new_result, download=download, extra_info=extra_info, params=params) elif result_type == 'playlist' or result_type == 'multi_video': # We process each entry in the playlist playlist = ie_result.get('title') or ie_result.get('id') @@ -843,7 +849,8 @@ class YoutubeDL(object): entry_result = self.process_ie_result(entry, download=download, - extra_info=extra) + extra_info=extra, + params=params) playlist_results.append(entry_result) ie_result['entries'] = playlist_results self.to_screen('[download] Finished downloading playlist: %s' % playlist) @@ -865,7 +872,7 @@ class YoutubeDL(object): ) return r ie_result['entries'] = [ - self.process_ie_result(_fixup(r), download, extra_info) + self.process_ie_result(_fixup(r), download, extra_info, params=params) for r in ie_result['entries'] ] return ie_result @@ -1213,7 +1220,7 @@ class YoutubeDL(object): self.cookiejar.add_cookie_header(pr) return pr.get_header('Cookie') - def process_video_result(self, info_dict, download=True): + def process_video_result(self, info_dict, params, download=True): assert info_dict.get('_type', 'video') == 'video' if 'id' not in info_dict: @@ -1283,7 +1290,7 @@ class YoutubeDL(object): return info_dict['requested_subtitles'] = self.process_subtitles( info_dict['id'], subtitles, - info_dict.get('automatic_captions')) + info_dict.get('automatic_captions'), params) # We now pick which formats have to be downloaded if info_dict.get('formats') is None: @@ -1352,10 +1359,10 @@ class YoutubeDL(object): self.list_formats(info_dict) return - req_format = self.params.get('format') + req_format = params.get('format') if req_format is None: req_format_list = [] - if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and + if (params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and not info_dict.get('is_live')): merger = FFmpegMergerPP(self) if merger.available and merger.can_merge(): @@ -1374,37 +1381,37 @@ class YoutubeDL(object): for format in formats_to_download: new_info = dict(info_dict) new_info.update(format) - self.process_info(new_info) + self.process_info(new_info, params) # We update the info dict with the best quality format (backwards compatibility) info_dict.update(formats_to_download[-1]) return info_dict - def process_subtitles(self, video_id, normal_subtitles, automatic_captions): + def process_subtitles(self, video_id, normal_subtitles, automatic_captions, params): """Select the requested subtitles and their format""" available_subs = {} - if normal_subtitles and self.params.get('writesubtitles'): + if normal_subtitles and params.get('writesubtitles'): available_subs.update(normal_subtitles) - if automatic_captions and self.params.get('writeautomaticsub'): + if automatic_captions and params.get('writeautomaticsub'): for lang, cap_info in automatic_captions.items(): if lang not in available_subs: available_subs[lang] = cap_info - if (not self.params.get('writesubtitles') and not - self.params.get('writeautomaticsub') or not + if (not params.get('writesubtitles') and not + params.get('writeautomaticsub') or not available_subs): return None - if self.params.get('allsubtitles', False): + if params.get('allsubtitles', False): requested_langs = available_subs.keys() else: - if self.params.get('subtitleslangs', False): - requested_langs = self.params.get('subtitleslangs') + if params.get('subtitleslangs', False): + requested_langs = params.get('subtitleslangs') elif 'en' in available_subs: requested_langs = ['en'] else: requested_langs = [list(available_subs.keys())[0]] - formats_query = self.params.get('subtitlesformat', 'best') + formats_query = params.get('subtitlesformat', 'best') formats_preference = formats_query.split('/') if formats_query else [] subs = {} for lang in requested_langs: @@ -1428,7 +1435,7 @@ class YoutubeDL(object): subs[lang] = f return subs - def process_info(self, info_dict): + def process_info(self, info_dict, params): """Process a single resolved IE result.""" assert info_dict.get('_type', 'video') == 'video' @@ -1452,7 +1459,7 @@ class YoutubeDL(object): self._num_downloads += 1 - info_dict['_filename'] = filename = self.prepare_filename(info_dict) + info_dict['_filename'] = filename = self.prepare_filename(info_dict, params) # Forced printings if self.params.get('forcetitle', False): @@ -1494,9 +1501,9 @@ class YoutubeDL(object): self.report_error('unable to create directory ' + error_to_compat_str(err)) return - if self.params.get('writedescription', False): + if params.get('writedescription', False): descfn = replace_extension(filename, 'description', info_dict.get('ext')) - if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)): + if params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)): self.to_screen('[info] Video description is already present') elif info_dict.get('description') is None: self.report_warning('There\'s no description to write.') @@ -1509,9 +1516,9 @@ class YoutubeDL(object): self.report_error('Cannot write description file ' + descfn) return - if self.params.get('writeannotations', False): + if params.get('writeannotations', False): annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext')) - if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)): + if params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)): self.to_screen('[info] Video annotations are already present') else: try: @@ -1524,8 +1531,8 @@ class YoutubeDL(object): self.report_error('Cannot write annotations file: ' + annofn) return - subtitles_are_requested = any([self.params.get('writesubtitles', False), - self.params.get('writeautomaticsub')]) + subtitles_are_requested = any([params.get('writesubtitles', False), + params.get('writeautomaticsub')]) if subtitles_are_requested and info_dict.get('requested_subtitles'): # subtitles download errors are already managed as troubles in relevant IE @@ -1546,7 +1553,7 @@ class YoutubeDL(object): continue try: sub_filename = subtitles_filename(filename, sub_lang, sub_format) - if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)): + if params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)): self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format)) else: self.to_screen('[info] Writing video subtitles to: ' + sub_filename) @@ -1556,9 +1563,9 @@ class YoutubeDL(object): self.report_error('Cannot write subtitles file ' + sub_filename) return - if self.params.get('writeinfojson', False): + if params.get('writeinfojson', False): infofn = replace_extension(filename, 'info.json', info_dict.get('ext')) - if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): + if params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): self.to_screen('[info] Video description metadata is already present') else: self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn) @@ -1570,10 +1577,10 @@ class YoutubeDL(object): self._write_thumbnails(info_dict, filename) - if not self.params.get('skip_download', False): + if not params.get('skip_download', False): try: def dl(name, info): - fd = get_suitable_downloader(info, self.params)(self, self.params) + fd = get_suitable_downloader(info, params)(self, params) for ph in self._progress_hooks: fd.add_progress_hook(ph) if self.params.get('verbose'): @@ -1613,7 +1620,7 @@ class YoutubeDL(object): if filename_real_ext == info_dict['ext'] else filename) requested_formats = info_dict['requested_formats'] - if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats): + if params.get('merge_output_format') is None and not compatible_formats(requested_formats): info_dict['ext'] = 'mkv' self.report_warning( 'Requested formats are incompatible for merge and will be merged into mkv.') @@ -1627,7 +1634,7 @@ class YoutubeDL(object): for f in requested_formats: new_info = dict(info_dict) new_info.update(f) - fname = self.prepare_filename(new_info) + fname = self.prepare_filename(new_info, params) fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext']) downloaded.append(fname) partial_success = dl(fname, new_info) @@ -1648,7 +1655,7 @@ class YoutubeDL(object): if success and filename != '-': # Fixup content - fixup_policy = self.params.get('fixup') + fixup_policy = params.get('fixup') if fixup_policy is None: fixup_policy = 'detect_or_warn' @@ -1693,7 +1700,7 @@ class YoutubeDL(object): if (info_dict.get('protocol') == 'm3u8_native' or info_dict.get('protocol') == 'm3u8' and - self.params.get('hls_prefer_native')): + params.get('hls_prefer_native')): if fixup_policy == 'warn': self.report_warning('%s: malformated aac bitstream.' % ( info_dict['id'])) @@ -1710,7 +1717,7 @@ class YoutubeDL(object): assert fixup_policy in ('ignore', 'never') try: - self.post_process(filename, info_dict) + self.post_process(filename, info_dict, params) except (PostProcessingError) as err: self.report_error('postprocessing: %s' % str(err)) return @@ -1747,7 +1754,11 @@ class YoutubeDL(object): # FileInput doesn't have a read method, we can't call json.load info = self.filter_requested_info(json.loads('\n'.join(f))) try: - self.process_ie_result(info, download=True) + params = None + ie_name = info.get('extractor') + if ie_name: + params = self.params.section(ie_name) + self.process_ie_result(info, download=True, params=params) except DownloadError: webpage_url = info.get('webpage_url') if webpage_url is not None: @@ -1763,7 +1774,7 @@ class YoutubeDL(object): (k, v) for k, v in info_dict.items() if k not in ['requested_formats', 'requested_subtitles']) - def post_process(self, filename, ie_info): + def post_process(self, filename, ie_info, params): """Run all the postprocessors on the given file.""" info = dict(ie_info) info['filepath'] = filename @@ -1777,7 +1788,7 @@ class YoutubeDL(object): files_to_delete, info = pp.run(info) except PostProcessingError as e: self.report_error(e.msg) - if files_to_delete and not self.params.get('keepvideo', False): + if files_to_delete and not params.get('keepvideo', False): for old_filename in files_to_delete: self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename) try: From e2887bb044357459c71df5265702c0ed12355341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 19 Mar 2016 12:49:12 +0100 Subject: [PATCH 04/11] [YoutubeDL] videos inherit parameters when they are contained in a playlist --- test/test_YoutubeDL.py | 19 ++++++++++++++++++- youtube_dl/YoutubeDL.py | 13 +++++++++---- youtube_dl/params.py | 7 +++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 86be58dcb..638979a05 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -710,6 +710,7 @@ class TestYoutubeDL(unittest.TestCase): ydl = YoutubeDL(params, auto_init=False) ydl.downloads = [] real_process_info = ydl.process_info + def process_info(info_dict, params): r = real_process_info(info_dict, params) ydl.downloads.append(info_dict) @@ -728,7 +729,6 @@ class TestYoutubeDL(unittest.TestCase): 'url': 'http://example.com', } - ie = ExampleIE() ydl.add_info_extractor(ie) pars = ie.params @@ -740,6 +740,23 @@ class TestYoutubeDL(unittest.TestCase): ydl.extract_info('example') self.assertEqual(ydl.downloads[-1]['_filename'], 'foo.mp4') + class ExamplePlaylistIE(InfoExtractor): + IE_NAME = 'example.com:playlist' + _VALID_URL = r'example:playlist' + + def _real_extract(self, url): + return { + '_type': 'playlist', + 'title': 'example playlist', + 'entries': [self.url_result('example')], + } + playlist_params = {'outtmpl': '%(playlist)s/%(title)s.%(ext)s'} + ydl.params = Params( + {'skip_download': True}, {'example.com:playlist': playlist_params}) + ydl.add_info_extractor(ExamplePlaylistIE()) + ydl.extract_info('example:playlist') + self.assertEqual(ydl.downloads[-1]['_filename'], 'example playlist/example.mp4') + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index e5e7482f6..8a1f56222 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -650,13 +650,16 @@ class YoutubeDL(object): info_dict.setdefault(key, value) def extract_info(self, url, download=True, ie_key=None, extra_info={}, - process=True, force_generic_extractor=False): + process=True, force_generic_extractor=False, params=None): ''' Returns a list with a dictionary for each video we find. If 'download', also downloads the videos. extra_info is a dict containing the extra values to add to each result ''' + if params is None: + params = self.params + if not ie_key and force_generic_extractor: ie_key = 'Generic' @@ -687,7 +690,7 @@ class YoutubeDL(object): if process: return self.process_ie_result( ie_result, download, extra_info, - params=self.params.section(ie.IE_NAME)) + params=params.section(ie.IE_NAME)) else: return ie_result except ExtractorError as e: # An error we somewhat expected @@ -741,12 +744,14 @@ class YoutubeDL(object): return self.extract_info(ie_result['url'], download, ie_key=ie_result.get('ie_key'), - extra_info=extra_info) + extra_info=extra_info, + params=params) elif result_type == 'url_transparent': # Use the information from the embedding page info = self.extract_info( ie_result['url'], ie_key=ie_result.get('ie_key'), - extra_info=extra_info, download=False, process=False) + extra_info=extra_info, download=False, process=False, + params=params) force_properties = dict( (k, v) for k, v in ie_result.items() if v is not None) diff --git a/youtube_dl/params.py b/youtube_dl/params.py index 03d2f2f2e..8b1e92d8d 100644 --- a/youtube_dl/params.py +++ b/youtube_dl/params.py @@ -41,3 +41,10 @@ class ParamsSection(object): return self[key] except KeyError: return default + + @property + def sections(self): + return self.parent.sections + + def section(self, section): + return ParamsSection(self.parent.sections.get(section, {}), self) From fa5bda9053daa255b61bb2d95cf72a3105aedb41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 19 Mar 2016 14:11:01 +0100 Subject: [PATCH 05/11] Refactor some parts of the option parsing code --- devscripts/bash-completion.py | 4 +- devscripts/fish-completion.py | 4 +- devscripts/zsh-completion.py | 4 +- youtube_dl/__init__.py | 154 +++++++++++++++++----------------- youtube_dl/options.py | 128 ++++++++++++++-------------- 5 files changed, 152 insertions(+), 142 deletions(-) diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index ce68f26f9..7a7bb37c1 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -6,7 +6,7 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl +from youtube_dl.options import build_option_parser BASH_COMPLETION_FILE = "youtube-dl.bash-completion" BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in" @@ -25,5 +25,5 @@ def build_completion(opt_parser): filled_template = template.replace("{{flags}}", " ".join(opts_flag)) f.write(filled_template) -parser = youtube_dl.parseOpts()[0] +parser = build_option_parser() build_completion(parser) diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index 41629d87d..5adfe4a58 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -7,7 +7,7 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl +from youtube_dl.options import build_option_parser from youtube_dl.utils import shell_quote FISH_COMPLETION_FILE = 'youtube-dl.fish' @@ -44,5 +44,5 @@ def build_completion(opt_parser): with open(FISH_COMPLETION_FILE, 'w') as f: f.write(filled_template) -parser = youtube_dl.parseOpts()[0] +parser = build_option_parser() build_completion(parser) diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 04728e8e2..48f53f5ac 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -6,7 +6,7 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl +from youtube_dl.options import build_option_parser ZSH_COMPLETION_FILE = "youtube-dl.zsh" ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in" @@ -44,5 +44,5 @@ def build_completion(opt_parser): with open(ZSH_COMPLETION_FILE, "w") as f: f.write(template) -parser = youtube_dl.parseOpts()[0] +parser = build_option_parser() build_completion(parser) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 737f6545d..0b26e7047 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -44,80 +44,7 @@ from .extractor import gen_extractors, list_extractors from .YoutubeDL import YoutubeDL -def _real_main(argv=None): - # Compatibility fixes for Windows - if sys.platform == 'win32': - # https://github.com/rg3/youtube-dl/issues/820 - codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) - - workaround_optparse_bug9161() - - setproctitle('youtube-dl') - - parser, opts, args = parseOpts(argv) - - # Set user agent - if opts.user_agent is not None: - std_headers['User-Agent'] = opts.user_agent - - # Set referer - if opts.referer is not None: - std_headers['Referer'] = opts.referer - - # Custom HTTP headers - if opts.headers is not None: - for h in opts.headers: - if h.find(':', 1) < 0: - parser.error('wrong header formatting, it should be key:value, not "%s"' % h) - key, value = h.split(':', 2) - if opts.verbose: - write_string('[debug] Adding header from command line option %s:%s\n' % (key, value)) - std_headers[key] = value - - # Dump user agent - if opts.dump_user_agent: - compat_print(std_headers['User-Agent']) - sys.exit(0) - - # Batch file verification - batch_urls = [] - if opts.batchfile is not None: - try: - if opts.batchfile == '-': - batchfd = sys.stdin - else: - batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore') - batch_urls = read_batch_urls(batchfd) - if opts.verbose: - write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n') - except IOError: - sys.exit('ERROR: batch file could not be read') - all_urls = batch_urls + args - all_urls = [url.strip() for url in all_urls] - _enc = preferredencoding() - all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls] - - if opts.list_extractors: - for ie in list_extractors(opts.age_limit): - compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) - matchedUrls = [url for url in all_urls if ie.suitable(url)] - for mu in matchedUrls: - compat_print(' ' + mu) - sys.exit(0) - if opts.list_extractor_descriptions: - for ie in list_extractors(opts.age_limit): - if not ie._WORKING: - continue - desc = getattr(ie, 'IE_DESC', ie.IE_NAME) - if desc is False: - continue - if hasattr(ie, 'SEARCH_KEY'): - _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow') - _COUNTS = ('', '5', '10', 'all') - desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES)) - compat_print(desc) - sys.exit(0) - +def _build_ydl_opts(opts, parser): # Conflicting, missing and erroneous options if opts.usenetrc and (opts.username is not None or opts.password is not None): parser.error('using .netrc conflicts with giving username/password') @@ -276,7 +203,7 @@ def _real_main(argv=None): None if opts.match_filter is None else match_filter_func(opts.match_filter)) - ydl_opts = { + return { 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, @@ -383,6 +310,83 @@ def _real_main(argv=None): 'cn_verification_proxy': opts.cn_verification_proxy, } + +def _real_main(argv=None): + # Compatibility fixes for Windows + if sys.platform == 'win32': + # https://github.com/rg3/youtube-dl/issues/820 + codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) + + workaround_optparse_bug9161() + + setproctitle('youtube-dl') + + parser, opts, args = parseOpts(argv) + + # Set user agent + if opts.user_agent is not None: + std_headers['User-Agent'] = opts.user_agent + + # Set referer + if opts.referer is not None: + std_headers['Referer'] = opts.referer + + # Custom HTTP headers + if opts.headers is not None: + for h in opts.headers: + if h.find(':', 1) < 0: + parser.error('wrong header formatting, it should be key:value, not "%s"' % h) + key, value = h.split(':', 2) + if opts.verbose: + write_string('[debug] Adding header from command line option %s:%s\n' % (key, value)) + std_headers[key] = value + + # Dump user agent + if opts.dump_user_agent: + compat_print(std_headers['User-Agent']) + sys.exit(0) + + # Batch file verification + batch_urls = [] + if opts.batchfile is not None: + try: + if opts.batchfile == '-': + batchfd = sys.stdin + else: + batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore') + batch_urls = read_batch_urls(batchfd) + if opts.verbose: + write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n') + except IOError: + sys.exit('ERROR: batch file could not be read') + all_urls = batch_urls + args + all_urls = [url.strip() for url in all_urls] + _enc = preferredencoding() + all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls] + + if opts.list_extractors: + for ie in list_extractors(opts.age_limit): + compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) + matchedUrls = [url for url in all_urls if ie.suitable(url)] + for mu in matchedUrls: + compat_print(' ' + mu) + sys.exit(0) + if opts.list_extractor_descriptions: + for ie in list_extractors(opts.age_limit): + if not ie._WORKING: + continue + desc = getattr(ie, 'IE_DESC', ie.IE_NAME) + if desc is False: + continue + if hasattr(ie, 'SEARCH_KEY'): + _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow') + _COUNTS = ('', '5', '10', 'all') + desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES)) + compat_print(desc) + sys.exit(0) + + ydl_opts = _build_ydl_opts(opts, parser) + with YoutubeDL(ydl_opts) as ydl: # Update version if opts.update_self: diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 7819f14ab..4f9e40c9d 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -19,57 +19,7 @@ from .utils import ( from .version import __version__ -def parseOpts(overrideArguments=None): - def _readOptions(filename_bytes, default=[]): - try: - optionf = open(filename_bytes) - except IOError: - return default # silently skip if file is not present - try: - res = [] - for l in optionf: - res += compat_shlex_split(l, comments=True) - finally: - optionf.close() - return res - - def _readUserConf(): - xdg_config_home = compat_getenv('XDG_CONFIG_HOME') - if xdg_config_home: - userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config') - if not os.path.isfile(userConfFile): - userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf') - else: - userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config') - if not os.path.isfile(userConfFile): - userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf') - userConf = _readOptions(userConfFile, None) - - if userConf is None: - appdata_dir = compat_getenv('appdata') - if appdata_dir: - userConf = _readOptions( - os.path.join(appdata_dir, 'youtube-dl', 'config'), - default=None) - if userConf is None: - userConf = _readOptions( - os.path.join(appdata_dir, 'youtube-dl', 'config.txt'), - default=None) - - if userConf is None: - userConf = _readOptions( - os.path.join(compat_expanduser('~'), 'youtube-dl.conf'), - default=None) - if userConf is None: - userConf = _readOptions( - os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'), - default=None) - - if userConf is None: - userConf = [] - - return userConf - +def build_option_parser(): def _format_option_string(option): ''' ('-o', '--option') -> -o, --format METAVAR''' @@ -90,16 +40,6 @@ def parseOpts(overrideArguments=None): def _comma_separated_values_options_callback(option, opt_str, value, parser): setattr(parser.values, option.dest, value.split(',')) - def _hide_login_info(opts): - opts = list(opts) - for private_opt in ['-p', '--password', '-u', '--username', '--video-password']: - try: - i = opts.index(private_opt) - opts[i + 1] = 'PRIVATE' - except ValueError: - pass - return opts - # No need to wrap help messages if we're on a wide console columns = compat_get_terminal_size().columns max_width = columns if columns else 80 @@ -786,6 +726,72 @@ def parseOpts(overrideArguments=None): parser.add_option_group(authentication) parser.add_option_group(postproc) + return parser + + +def parseOpts(overrideArguments=None): + def _readOptions(filename_bytes, default=[]): + try: + optionf = open(filename_bytes) + except IOError: + return default # silently skip if file is not present + try: + res = [] + for l in optionf: + res += compat_shlex_split(l, comments=True) + finally: + optionf.close() + return res + + def _readUserConf(): + xdg_config_home = compat_getenv('XDG_CONFIG_HOME') + if xdg_config_home: + userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config') + if not os.path.isfile(userConfFile): + userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf') + else: + userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config') + if not os.path.isfile(userConfFile): + userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf') + userConf = _readOptions(userConfFile, None) + + if userConf is None: + appdata_dir = compat_getenv('appdata') + if appdata_dir: + userConf = _readOptions( + os.path.join(appdata_dir, 'youtube-dl', 'config'), + default=None) + if userConf is None: + userConf = _readOptions( + os.path.join(appdata_dir, 'youtube-dl', 'config.txt'), + default=None) + + if userConf is None: + userConf = _readOptions( + os.path.join(compat_expanduser('~'), 'youtube-dl.conf'), + default=None) + if userConf is None: + userConf = _readOptions( + os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'), + default=None) + + if userConf is None: + userConf = [] + + return userConf + + def _hide_login_info(opts): + opts = list(opts) + for private_opt in ['-p', '--password', '-u', '--username', '--video-password']: + try: + i = opts.index(private_opt) + opts[i + 1] = 'PRIVATE' + except ValueError: + pass + return opts + + parser = build_option_parser() + if overrideArguments is not None: opts, args = parser.parse_args(overrideArguments) if opts.verbose: From e6f20cd14253bfb1774027f1c76f5edf66ede640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 19 Mar 2016 16:39:20 +0100 Subject: [PATCH 06/11] Add support for loading a config file with different parameters for each site On Unix it look for ~/.config/youtube-dl/config.ini or ~/.config/youtube-dl.ini, Windows support is currently missing. The new config file is in the INI format, the options are given in the long form without the leading dashes. Parameters for an specific extractor are written in their own section. When a video is contained in a playlist, it inherits the parameters from the playlist extractor. Example: write-thumbnail= [youtube:playlist] output=%(playlist)s/%(id)s.%(ext)s [youtube] format=webm All the videos use '--write-thumbnail', videos inside a YouTube playlist use the specified output template and YouTube videos are downloaded in the webm format. --- youtube_dl/__init__.py | 29 +++++++++++++++++++++++-- youtube_dl/compat.py | 6 ++++++ youtube_dl/options.py | 48 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 0b26e7047..e78b49236 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -42,6 +42,7 @@ from .downloader import ( ) from .extractor import gen_extractors, list_extractors from .YoutubeDL import YoutubeDL +from .params import Params, ParamsSection def _build_ydl_opts(opts, parser): @@ -311,6 +312,30 @@ def _build_ydl_opts(opts, parser): } +class CLIParams(Params): + def __init__(self, opts, build_section_args, parser): + super(CLIParams, self).__init__(_build_ydl_opts(opts, parser)) + self.build_section_args = build_section_args + self.parser = parser + + def section(self, section): + return CLIParamsSection([section], self.build_section_args, self.parser) + + +# TODO: investigate if it's worth to cache this +class CLIParamsSection(ParamsSection): + def __init__(self, sections, build_section_args, parser): + section_args = build_section_args(*sections) + section_opts = parser.parse_args(section_args)[0] + super(CLIParamsSection, self).__init__(_build_ydl_opts(section_opts, parser), {}) + self.section_names = sections + self.build_section_args = build_section_args + self.parser = parser + + def section(self, section): + return CLIParamsSection(self.section_names + [section], self.build_section_args, self.parser) + + def _real_main(argv=None): # Compatibility fixes for Windows if sys.platform == 'win32': @@ -321,7 +346,7 @@ def _real_main(argv=None): setproctitle('youtube-dl') - parser, opts, args = parseOpts(argv) + parser, opts, build_section_args, args = parseOpts(argv) # Set user agent if opts.user_agent is not None: @@ -385,7 +410,7 @@ def _real_main(argv=None): compat_print(desc) sys.exit(0) - ydl_opts = _build_ydl_opts(opts, parser) + ydl_opts = CLIParams(opts, build_section_args, parser) with YoutubeDL(ydl_opts) as ydl: # Update version diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 76b6b0e38..feb52fd56 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -582,11 +582,17 @@ if sys.version_info >= (3, 0): else: from tokenize import generate_tokens as compat_tokenize_tokenize +try: + import configparser as compat_configparser +except ImportError: + import ConfigParser as compat_configparser + __all__ = [ 'compat_HTMLParser', 'compat_HTTPError', 'compat_basestring', 'compat_chr', + 'compat_configparser', 'compat_cookiejar', 'compat_cookies', 'compat_etree_fromstring', diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 4f9e40c9d..cbc752190 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -1,11 +1,13 @@ from __future__ import unicode_literals +import io import os.path import optparse import sys from .downloader.external import list_external_downloaders from .compat import ( + compat_configparser, compat_expanduser, compat_get_terminal_size, compat_getenv, @@ -743,17 +745,41 @@ def parseOpts(overrideArguments=None): optionf.close() return res + def _readIni(filename_bytes, default=([], {})): + try: + ini = open(filename_bytes) + except IOError: + return default # silently skip if file is not present + parser = compat_configparser.RawConfigParser() + ini = io.StringIO('[@GLOBAL@]\n' + ini.read()) + parser.readfp(ini) + + def convert_opts(opts): + return [ + '--' + opt + ('' if not arg else ('=' + arg)) + for opt, arg in opts] + global_opts = convert_opts(parser.items('@GLOBAL@')) + section_opts = dict((section, convert_opts(parser.items(section))) for section in parser.sections() if section != '@GLOBAL@') + return global_opts, section_opts + def _readUserConf(): xdg_config_home = compat_getenv('XDG_CONFIG_HOME') if xdg_config_home: userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config') + userIniFile = os.path.join(xdg_config_home, 'youtube-dl', 'config.ini') if not os.path.isfile(userConfFile): userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf') + if not os.path.isfile(userIniFile): + userIniFile = os.path.join(xdg_config_home, 'youtube-dl.ini') else: userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config') + userIniFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config.ini') if not os.path.isfile(userConfFile): userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf') + if not os.path.isfile(userIniFile): + userIniFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.ini') userConf = _readOptions(userConfFile, None) + userIni = _readIni(userIniFile, None) if userConf is None: appdata_dir = compat_getenv('appdata') @@ -778,7 +804,11 @@ def parseOpts(overrideArguments=None): if userConf is None: userConf = [] - return userConf + if userIni is None: + userIni = [], {} + iniGlobal, iniSections = userIni + + return userConf + iniGlobal, iniSections def _hide_login_info(opts): opts = list(opts) @@ -807,18 +837,30 @@ def parseOpts(overrideArguments=None): if '--ignore-config' in command_line_conf: system_conf = [] user_conf = [] + user_sections = {} else: system_conf = compat_conf(_readOptions('/etc/youtube-dl.conf')) if '--ignore-config' in system_conf: user_conf = [] + user_sections = {} else: - user_conf = compat_conf(_readUserConf()) + user_conf, user_sections = _readUserConf() + user_conf = compat_conf(user_conf) argv = system_conf + user_conf + command_line_conf opts, args = parser.parse_args(argv) if opts.verbose: write_string('[debug] System config: ' + repr(_hide_login_info(system_conf)) + '\n') write_string('[debug] User config: ' + repr(_hide_login_info(user_conf)) + '\n') + for section, section_args in user_sections.items(): + write_string('[debug] User config for "' + section + '": ' + repr(_hide_login_info(section_args)) + '\n') write_string('[debug] Command-line args: ' + repr(_hide_login_info(command_line_conf)) + '\n') - return parser, opts, args + def build_section_args(*sections): + res = system_conf + user_conf + for section in sections: + res.extend(user_sections.get(section, [])) + res.extend(command_line_conf) + return res + + return parser, opts, build_section_args, args From 57e1fb653283a58c178b4f95f6ac69c25010517c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 22 Mar 2016 23:26:29 +0100 Subject: [PATCH 07/11] [YoutubeDL] allow to use different postprocessors for each extractor 'add_post_processor' is kept for backwards compatibility, but it should probably be eventually removed. --- youtube_dl/YoutubeDL.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 8a1f56222..0665610c8 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -355,13 +355,6 @@ class YoutubeDL(object): self.print_debug_header() self.add_default_info_extractors() - for pp_def_raw in self.params.get('postprocessors', []): - pp_class = get_postprocessor(pp_def_raw['key']) - pp_def = dict(pp_def_raw) - del pp_def['key'] - pp = pp_class(self, **compat_kwargs(pp_def)) - self.add_post_processor(pp) - for ph in self.params.get('progress_hooks', []): self.add_progress_hook(ph) @@ -1786,6 +1779,13 @@ class YoutubeDL(object): pps_chain = [] if ie_info.get('__postprocessors') is not None: pps_chain.extend(ie_info['__postprocessors']) + for pp_def_raw in params.get('postprocessors', []): + pp_class = get_postprocessor(pp_def_raw['key']) + pp_def = dict(pp_def_raw) + del pp_def['key'] + pp = pp_class(self, **compat_kwargs(pp_def)) + pp.set_downloader(self) + pps_chain.append(pp) pps_chain.extend(self._pps) for pp in pps_chain: files_to_delete = [] From 7fada6e36e29aad3af3bd6b5357a0ae75501a425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 23 Mar 2016 10:56:12 +0100 Subject: [PATCH 08/11] [options] Read ini files in windows --- youtube_dl/options.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index cbc752190..0d7340b4e 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -781,16 +781,21 @@ def parseOpts(overrideArguments=None): userConf = _readOptions(userConfFile, None) userIni = _readIni(userIniFile, None) - if userConf is None: + if userConf is None or userIni is None: appdata_dir = compat_getenv('appdata') if appdata_dir: - userConf = _readOptions( - os.path.join(appdata_dir, 'youtube-dl', 'config'), - default=None) + if userConf is None: + userConf = _readOptions( + os.path.join(appdata_dir, 'youtube-dl', 'config'), + default=None) if userConf is None: userConf = _readOptions( os.path.join(appdata_dir, 'youtube-dl', 'config.txt'), default=None) + if userIni is None: + userIni = _readIni( + os.path.join(appdata_dir, 'youtube-dl', 'config.ini'), + default=None) if userConf is None: userConf = _readOptions( @@ -801,6 +806,15 @@ def parseOpts(overrideArguments=None): os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'), default=None) + if userIni is None: + userIni = _readIni( + os.path.join(compat_expanduser('~'), 'youtube-dl.ini'), + default=None) + if userIni is None: + userIni = _readIni( + os.path.join(compat_expanduser('~'), 'youtube-dl.ini.txt'), + default=None) + if userConf is None: userConf = [] From 14d77aaab2fcb632832afcdcd37de0bb22158026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 1 Apr 2016 14:01:45 +0200 Subject: [PATCH 09/11] Read /etc/youtube-dl.ini config file --- youtube_dl/options.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 0d7340b4e..8c2a37214 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -854,6 +854,9 @@ def parseOpts(overrideArguments=None): user_sections = {} else: system_conf = compat_conf(_readOptions('/etc/youtube-dl.conf')) + system_global, system_sections = _readIni('/etc/youtube-dl.ini') + system_conf.extend(system_global) + if '--ignore-config' in system_conf: user_conf = [] user_sections = {} @@ -865,6 +868,8 @@ def parseOpts(overrideArguments=None): opts, args = parser.parse_args(argv) if opts.verbose: write_string('[debug] System config: ' + repr(_hide_login_info(system_conf)) + '\n') + for section, section_args in system_sections.items(): + write_string('[debug] System config for "' + section + '": ' + repr(_hide_login_info(section_args)) + '\n') write_string('[debug] User config: ' + repr(_hide_login_info(user_conf)) + '\n') for section, section_args in user_sections.items(): write_string('[debug] User config for "' + section + '": ' + repr(_hide_login_info(section_args)) + '\n') @@ -873,6 +878,7 @@ def parseOpts(overrideArguments=None): def build_section_args(*sections): res = system_conf + user_conf for section in sections: + res.extend(system_sections.get(section, [])) res.extend(user_sections.get(section, [])) res.extend(command_line_conf) return res From 479181e3a646131997860e6349f8c53327084d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 1 Apr 2016 14:30:12 +0200 Subject: [PATCH 10/11] README.md: document how to set specific options for each extractor --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index e972bf69f..e986ee010 100644 --- a/README.md +++ b/README.md @@ -427,6 +427,24 @@ Note that options in configuration file are just the same options aka switches u You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run. +### Per extractor configuration + +In addition to globally setting options, you can also set different options for each extractor. +A different set of files is used: `~/.config/youtube-dl/config.ini` and `/etc/youtube-dl.ini` on Unix, `%APPDATA%\youtube-dl\config.ini` or `C:\Users\\youtube-dl.ini` on Windows. +The files are stored in the [INI format](https://en.wikipedia.org/wiki/INI_file), each argument must be written in its own line using its full name without the leading `--`. +To start a section with the options for an specific extractor you can write a line in the form `[]`, where `` is the name printed before the info message while downloading (like `youtube`, `youtube:playlist`, `vimeo` ...). + +For example, with the following configuration file youtube-dl will always extract the audio, as an mp3 file for YouTube videos and if the are inside a playlist they will be saved in a different folder: +``` +extract-audio= + +[youtube] +audio-format=mp3 + +[youtube:playlist] +output=%(playlist)s/%(title)s.%(ext)s +``` + ### Authentication with `.netrc` file You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only: From 648f8895511608f7b57b245e772f5fd535fde253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 1 Apr 2016 20:50:35 +0200 Subject: [PATCH 11/11] Support using the short option name in the INI configuration files --- README.md | 4 ++-- youtube_dl/options.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e986ee010..93aa08c58 100644 --- a/README.md +++ b/README.md @@ -431,12 +431,12 @@ You can use `--ignore-config` if you want to disable the configuration file for In addition to globally setting options, you can also set different options for each extractor. A different set of files is used: `~/.config/youtube-dl/config.ini` and `/etc/youtube-dl.ini` on Unix, `%APPDATA%\youtube-dl\config.ini` or `C:\Users\\youtube-dl.ini` on Windows. -The files are stored in the [INI format](https://en.wikipedia.org/wiki/INI_file), each argument must be written in its own line using its full name without the leading `--`. +The files are stored in the [INI format](https://en.wikipedia.org/wiki/INI_file), each argument must be written in its own line using its full name without the leading `--` or the short version without the leading `-`. To start a section with the options for an specific extractor you can write a line in the form `[]`, where `` is the name printed before the info message while downloading (like `youtube`, `youtube:playlist`, `vimeo` ...). For example, with the following configuration file youtube-dl will always extract the audio, as an mp3 file for YouTube videos and if the are inside a playlist they will be saved in a different folder: ``` -extract-audio= +x= [youtube] audio-format=mp3 diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 8c2a37214..a97fc7ae4 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -756,7 +756,7 @@ def parseOpts(overrideArguments=None): def convert_opts(opts): return [ - '--' + opt + ('' if not arg else ('=' + arg)) + ('-' if len(opt) == 1 else '--') + opt + ('' if not arg else ('=' + arg)) for opt, arg in opts] global_opts = convert_opts(parser.items('@GLOBAL@')) section_opts = dict((section, convert_opts(parser.items(section))) for section in parser.sections() if section != '@GLOBAL@')