From fa5bda9053daa255b61bb2d95cf72a3105aedb41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 19 Mar 2016 14:11:01 +0100 Subject: [PATCH] Refactor some parts of the option parsing code --- devscripts/bash-completion.py | 4 +- devscripts/fish-completion.py | 4 +- devscripts/zsh-completion.py | 4 +- youtube_dl/__init__.py | 154 +++++++++++++++++----------------- youtube_dl/options.py | 128 ++++++++++++++-------------- 5 files changed, 152 insertions(+), 142 deletions(-) diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index ce68f26f9..7a7bb37c1 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -6,7 +6,7 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl +from youtube_dl.options import build_option_parser BASH_COMPLETION_FILE = "youtube-dl.bash-completion" BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in" @@ -25,5 +25,5 @@ def build_completion(opt_parser): filled_template = template.replace("{{flags}}", " ".join(opts_flag)) f.write(filled_template) -parser = youtube_dl.parseOpts()[0] +parser = build_option_parser() build_completion(parser) diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index 41629d87d..5adfe4a58 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -7,7 +7,7 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl +from youtube_dl.options import build_option_parser from youtube_dl.utils import shell_quote FISH_COMPLETION_FILE = 'youtube-dl.fish' @@ -44,5 +44,5 @@ def build_completion(opt_parser): with open(FISH_COMPLETION_FILE, 'w') as f: f.write(filled_template) -parser = youtube_dl.parseOpts()[0] +parser = build_option_parser() build_completion(parser) diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 04728e8e2..48f53f5ac 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -6,7 +6,7 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl +from youtube_dl.options import build_option_parser ZSH_COMPLETION_FILE = "youtube-dl.zsh" ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in" @@ -44,5 +44,5 @@ def build_completion(opt_parser): with open(ZSH_COMPLETION_FILE, "w") as f: f.write(template) -parser = youtube_dl.parseOpts()[0] +parser = build_option_parser() build_completion(parser) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 737f6545d..0b26e7047 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -44,80 +44,7 @@ from .extractor import gen_extractors, list_extractors from .YoutubeDL import YoutubeDL -def _real_main(argv=None): - # Compatibility fixes for Windows - if sys.platform == 'win32': - # https://github.com/rg3/youtube-dl/issues/820 - codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) - - workaround_optparse_bug9161() - - setproctitle('youtube-dl') - - parser, opts, args = parseOpts(argv) - - # Set user agent - if opts.user_agent is not None: - std_headers['User-Agent'] = opts.user_agent - - # Set referer - if opts.referer is not None: - std_headers['Referer'] = opts.referer - - # Custom HTTP headers - if opts.headers is not None: - for h in opts.headers: - if h.find(':', 1) < 0: - parser.error('wrong header formatting, it should be key:value, not "%s"' % h) - key, value = h.split(':', 2) - if opts.verbose: - write_string('[debug] Adding header from command line option %s:%s\n' % (key, value)) - std_headers[key] = value - - # Dump user agent - if opts.dump_user_agent: - compat_print(std_headers['User-Agent']) - sys.exit(0) - - # Batch file verification - batch_urls = [] - if opts.batchfile is not None: - try: - if opts.batchfile == '-': - batchfd = sys.stdin - else: - batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore') - batch_urls = read_batch_urls(batchfd) - if opts.verbose: - write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n') - except IOError: - sys.exit('ERROR: batch file could not be read') - all_urls = batch_urls + args - all_urls = [url.strip() for url in all_urls] - _enc = preferredencoding() - all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls] - - if opts.list_extractors: - for ie in list_extractors(opts.age_limit): - compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) - matchedUrls = [url for url in all_urls if ie.suitable(url)] - for mu in matchedUrls: - compat_print(' ' + mu) - sys.exit(0) - if opts.list_extractor_descriptions: - for ie in list_extractors(opts.age_limit): - if not ie._WORKING: - continue - desc = getattr(ie, 'IE_DESC', ie.IE_NAME) - if desc is False: - continue - if hasattr(ie, 'SEARCH_KEY'): - _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow') - _COUNTS = ('', '5', '10', 'all') - desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES)) - compat_print(desc) - sys.exit(0) - +def _build_ydl_opts(opts, parser): # Conflicting, missing and erroneous options if opts.usenetrc and (opts.username is not None or opts.password is not None): parser.error('using .netrc conflicts with giving username/password') @@ -276,7 +203,7 @@ def _real_main(argv=None): None if opts.match_filter is None else match_filter_func(opts.match_filter)) - ydl_opts = { + return { 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, @@ -383,6 +310,83 @@ def _real_main(argv=None): 'cn_verification_proxy': opts.cn_verification_proxy, } + +def _real_main(argv=None): + # Compatibility fixes for Windows + if sys.platform == 'win32': + # https://github.com/rg3/youtube-dl/issues/820 + codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) + + workaround_optparse_bug9161() + + setproctitle('youtube-dl') + + parser, opts, args = parseOpts(argv) + + # Set user agent + if opts.user_agent is not None: + std_headers['User-Agent'] = opts.user_agent + + # Set referer + if opts.referer is not None: + std_headers['Referer'] = opts.referer + + # Custom HTTP headers + if opts.headers is not None: + for h in opts.headers: + if h.find(':', 1) < 0: + parser.error('wrong header formatting, it should be key:value, not "%s"' % h) + key, value = h.split(':', 2) + if opts.verbose: + write_string('[debug] Adding header from command line option %s:%s\n' % (key, value)) + std_headers[key] = value + + # Dump user agent + if opts.dump_user_agent: + compat_print(std_headers['User-Agent']) + sys.exit(0) + + # Batch file verification + batch_urls = [] + if opts.batchfile is not None: + try: + if opts.batchfile == '-': + batchfd = sys.stdin + else: + batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore') + batch_urls = read_batch_urls(batchfd) + if opts.verbose: + write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n') + except IOError: + sys.exit('ERROR: batch file could not be read') + all_urls = batch_urls + args + all_urls = [url.strip() for url in all_urls] + _enc = preferredencoding() + all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls] + + if opts.list_extractors: + for ie in list_extractors(opts.age_limit): + compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) + matchedUrls = [url for url in all_urls if ie.suitable(url)] + for mu in matchedUrls: + compat_print(' ' + mu) + sys.exit(0) + if opts.list_extractor_descriptions: + for ie in list_extractors(opts.age_limit): + if not ie._WORKING: + continue + desc = getattr(ie, 'IE_DESC', ie.IE_NAME) + if desc is False: + continue + if hasattr(ie, 'SEARCH_KEY'): + _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow') + _COUNTS = ('', '5', '10', 'all') + desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES)) + compat_print(desc) + sys.exit(0) + + ydl_opts = _build_ydl_opts(opts, parser) + with YoutubeDL(ydl_opts) as ydl: # Update version if opts.update_self: diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 7819f14ab..4f9e40c9d 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -19,57 +19,7 @@ from .utils import ( from .version import __version__ -def parseOpts(overrideArguments=None): - def _readOptions(filename_bytes, default=[]): - try: - optionf = open(filename_bytes) - except IOError: - return default # silently skip if file is not present - try: - res = [] - for l in optionf: - res += compat_shlex_split(l, comments=True) - finally: - optionf.close() - return res - - def _readUserConf(): - xdg_config_home = compat_getenv('XDG_CONFIG_HOME') - if xdg_config_home: - userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config') - if not os.path.isfile(userConfFile): - userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf') - else: - userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config') - if not os.path.isfile(userConfFile): - userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf') - userConf = _readOptions(userConfFile, None) - - if userConf is None: - appdata_dir = compat_getenv('appdata') - if appdata_dir: - userConf = _readOptions( - os.path.join(appdata_dir, 'youtube-dl', 'config'), - default=None) - if userConf is None: - userConf = _readOptions( - os.path.join(appdata_dir, 'youtube-dl', 'config.txt'), - default=None) - - if userConf is None: - userConf = _readOptions( - os.path.join(compat_expanduser('~'), 'youtube-dl.conf'), - default=None) - if userConf is None: - userConf = _readOptions( - os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'), - default=None) - - if userConf is None: - userConf = [] - - return userConf - +def build_option_parser(): def _format_option_string(option): ''' ('-o', '--option') -> -o, --format METAVAR''' @@ -90,16 +40,6 @@ def parseOpts(overrideArguments=None): def _comma_separated_values_options_callback(option, opt_str, value, parser): setattr(parser.values, option.dest, value.split(',')) - def _hide_login_info(opts): - opts = list(opts) - for private_opt in ['-p', '--password', '-u', '--username', '--video-password']: - try: - i = opts.index(private_opt) - opts[i + 1] = 'PRIVATE' - except ValueError: - pass - return opts - # No need to wrap help messages if we're on a wide console columns = compat_get_terminal_size().columns max_width = columns if columns else 80 @@ -786,6 +726,72 @@ def parseOpts(overrideArguments=None): parser.add_option_group(authentication) parser.add_option_group(postproc) + return parser + + +def parseOpts(overrideArguments=None): + def _readOptions(filename_bytes, default=[]): + try: + optionf = open(filename_bytes) + except IOError: + return default # silently skip if file is not present + try: + res = [] + for l in optionf: + res += compat_shlex_split(l, comments=True) + finally: + optionf.close() + return res + + def _readUserConf(): + xdg_config_home = compat_getenv('XDG_CONFIG_HOME') + if xdg_config_home: + userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config') + if not os.path.isfile(userConfFile): + userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf') + else: + userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config') + if not os.path.isfile(userConfFile): + userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf') + userConf = _readOptions(userConfFile, None) + + if userConf is None: + appdata_dir = compat_getenv('appdata') + if appdata_dir: + userConf = _readOptions( + os.path.join(appdata_dir, 'youtube-dl', 'config'), + default=None) + if userConf is None: + userConf = _readOptions( + os.path.join(appdata_dir, 'youtube-dl', 'config.txt'), + default=None) + + if userConf is None: + userConf = _readOptions( + os.path.join(compat_expanduser('~'), 'youtube-dl.conf'), + default=None) + if userConf is None: + userConf = _readOptions( + os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'), + default=None) + + if userConf is None: + userConf = [] + + return userConf + + def _hide_login_info(opts): + opts = list(opts) + for private_opt in ['-p', '--password', '-u', '--username', '--video-password']: + try: + i = opts.index(private_opt) + opts[i + 1] = 'PRIVATE' + except ValueError: + pass + return opts + + parser = build_option_parser() + if overrideArguments is not None: opts, args = parser.parse_args(overrideArguments) if opts.verbose: