From 7bbaacbb9faf0a8ed71ddf0712b75e568206504f Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 20:50:39 -0400 Subject: [PATCH 01/35] Adding the documentation in the docstring, replaced optparse import with docopt --- youtube_dl/__init__.py | 192 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 189 insertions(+), 3 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 696e54f49..6b0570973 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -1,5 +1,189 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +""" +Usage: + youtube-dl [--help] [--version] [--update] [--ignore-errors] + [--dump-user-agent] [--user-agent UA] [--referer REF] + [--list-extractors] [--extractor-descriptions] [--proxy URL] + [--no-check-certificate] [--cache-dir] [--no-cache-dir] + [--playlist-start NUMBER] [--playlist-end NUMBER] + [--match-title REGEX] [--reject-title REGEX] + [--max-downloads NUMBER] [--min-filesize SIZE] + [--max-filesize SIZE] [--date DATE] [--datebefore DATE] + [--dateafter DATE] [--no-playlist] [--rate-limit LIMIT] + [--retries RETRIES] [--buffer-size SIZE] [--no-resize-buffer] + [--title] [--id] [--literal] [--auto-number] [--output TEMPLATE] + [--autonumber-size NUMBER] [--restrict-filenames] + [--batch-file FILE] [--no-overwrites] [--continue] [--no-continue] + [--cookies FILE] [--no-part] [--no-mtime] [--write-description] + [--write-info-json] [--write-thumbnail] [--quiet] [--simulate] + [--skip-download] [--get-url] [--get-title] [--get-id] + [--get-thumbnail] [--get-description] [--get-filename] + [--get-format] [--newline] [--no-progress] [--console-title] + [--verbose] [--dump-intermediate-pages] [--format FORMAT] + [--all-formats] [--prefer-free-formats] [--max-quality FORMAT] + [--list-formats] [--write-sub] [--write-auto-sub] [--all-subs] + [--list-subs] [--sub-format FORMAT] [--sub-lang LANGS] + [--username USERNAME] [--password PASSWORD] [--netrc] + [--video-password PASSWORD] [--extract-audio] + [--audio-format FORMAT] [--audio-quality QUALITY] + [--recode-video FORMAT] [--keep-video] [--no-post-overwrites] + [--embed-subs] + URL [url...] + +Options: + General Options: + -h, --help print this help text and exit + --version print program version and exit + -U, --update update this program to latest version. Make sure + that you have sufficient permissions (run with + sudo if needed) + -i, --ignore-errors continue on download errors, for example to to + skip unavailable videos in a playlist + --dump-user-agent display the current browser identification + --user-agent UA specify a custom user agent + --referer REF specify a custom referer, use if the video access + is restricted to one domain + --list-extractors List all supported extractors and the URLs they + would handle + --extractor-descriptions Output descriptions of all supported extractors + --proxy URL Use the specified HTTP/HTTPS proxy + --no-check-certificate Suppress HTTPS certificate validation. + --cache-dir None Location in the filesystem where youtube-dl can + store downloaded information permanently. By + default $XDG_CACHE_HOME/youtube-dl or ~/.cache + /youtube-dl . + --no-cache-dir Disable filesystem caching + + Video Selection: + --playlist-start NUMBER playlist video to start at [default: 1] + --playlist-end NUMBER playlist video to end at (default is last) + --match-title REGEX download only matching titles (regex or caseless + sub-string) + --reject-title REGEX skip download for matching titles (regex or + caseless sub-string) + --max-downloads NUMBER Abort after downloading NUMBER files + --min-filesize SIZE Do not download any videos smaller than SIZE + (e.g. 50k or 44.6m) + --max-filesize SIZE Do not download any videos larger than SIZE (e.g. + 50k or 44.6m) + --date DATE download only videos uploaded in this date + --datebefore DATE download only videos uploaded before this date + --dateafter DATE download only videos uploaded after this date + --no-playlist download only the currently playing video + + Download Options: + -r, --rate-limit LIMIT maximum download rate (e.g. 50k or 44.6m) + -R, --retries RETRIES number of retries [default: 10] + --buffer-size SIZE size of download buffer (e.g. 1024 or 16k) + (default is 1024) + --no-resize-buffer do not automatically adjust the buffer size. By + default, the buffer size is automatically resized + from an initial value of SIZE. + + Filesystem Options: + -t, --title use title in file name (default) + --id use only video ID in file name + -l, --literal [deprecated] alias of --title + -A, --auto-number number downloaded files starting from 00000 + -o, --output TEMPLATE output filename template. Use %(title)s to get + the title, %(uploader)s for the uploader name, + %(uploader_id)s for the uploader nickname if + different, %(autonumber)s to get an automatically + incremented number, %(ext)s for the filename + extension, %(upload_date)s for the upload date + (YYYYMMDD), %(extractor)s for the provider + (youtube, metacafe, etc), %(id)s for the video id + , %(playlist)s for the playlist the video is in, + %(playlist_index)s for the position in the + playlist and %% for a literal percent. Use - to + output to stdout. Can also be used to download to + a different directory, for example with -o '/my/d + ownloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' . + --autonumber-size NUMBER Specifies the number of digits in %(autonumber)s + when it is present in output filename template or + --autonumber option is given + --restrict-filenames Restrict filenames to only ASCII characters, and + avoid "&" and spaces in filenames + -a, --batch-file FILE file containing URLs to download ('-' for stdin) + -w, --no-overwrites do not overwrite files + -c, --continue resume partially downloaded files + --no-continue do not resume partially downloaded files (restart + from beginning) + --cookies FILE file to read cookies from and dump cookie jar in + --no-part do not use .part files + --no-mtime do not use the Last-modified header to set the + file modification time + --write-description write video description to a .description file + --write-info-json write video metadata to a .info.json file + --write-thumbnail write thumbnail image to disk + + Verbosity / Simulation Options: + -q, --quiet activates quiet mode + -s, --simulate do not download the video and do not write + anything to disk + --skip-download do not download the video + -g, --get-url simulate, quiet but print URL + -e, --get-title simulate, quiet but print title + --get-id simulate, quiet but print id + --get-thumbnail simulate, quiet but print thumbnail URL + --get-description simulate, quiet but print video description + --get-filename simulate, quiet but print output filename + --get-format simulate, quiet but print output format + --newline output progress bar as new lines + --no-progress do not print progress bar + --console-title display progress in console titlebar + -v, --verbose print various debugging information + --dump-intermediate-pages print downloaded pages to debug problems(very + verbose) + + Video Format Options: + -f, --format FORMAT video format code, specifiy the order of + preference using slashes: "-f 22/17/18". "-f mp4" + and "-f flv" are also supported + --all-formats download all available video formats + --prefer-free-formats prefer free video formats unless a specific one + is requested + --max-quality FORMAT highest quality format to download + -F, --list-formats list all available formats (currently youtube + only) + + Subtitle Options: + --write-sub write subtitle file + --write-auto-sub write automatic subtitle file (youtube only) + --all-subs downloads all the available subtitles of the + video + --list-subs lists all available subtitles for the video + --sub-format FORMAT subtitle format (default=srt) ([sbv/vtt] youtube + only) + --sub-lang LANGS languages of the subtitles to download (optional) + separated by commas, use IETF language tags like + 'en,pt' + + Authentication Options: + -u, --username USERNAME account username + -p, --password PASSWORD account password + -n, --netrc use .netrc authentication data + --video-password PASSWORD video password (vimeo only) + + Post-processing Options: + -x, --extract-audio convert video files to audio-only files (requires + ffmpeg or avconv and ffprobe or avprobe) + --audio-format FORMAT "best", "aac", "vorbis", "mp3", "m4a", "opus", or + "wav"; [default: best] + --audio-quality QUALITY ffmpeg/avconv audio quality specification, insert + a value between 0 (better) and 9 (worse) for VBR + or a specific bitrate like 128K [default: 5] + --recode-video FORMAT Encode the video to another format if necessary + (currently supported: mp4|flv|ogg|webm) + -k, --keep-video keeps the video file on disk after the post- + processing; the video is erased by default + --no-post-overwrites do not overwrite post-processed files; the post- + processed files are overwritten by default + --embed-subs embed subtitles in the video (only for mp4 + videos) +""" + __authors__ = ( 'Ricardo Garcia Gonzalez', @@ -30,13 +214,14 @@ __authors__ = ( 'Pierre Rudloff', 'Huarong Huo', 'Ismael Mejía', + 'Pablo Barton' ) __license__ = 'Public Domain' import codecs import getpass -import optparse +#import optparse import os import random import re @@ -47,7 +232,7 @@ import sys import warnings import platform - +from .docopt import docopt from .utils import * from .update import update_self from .version import __version__ @@ -379,7 +564,8 @@ def _real_main(argv=None): # https://github.com/rg3/youtube-dl/issues/820 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) - parser, opts, args = parseOpts(argv) + opts = docopt(__doc__, version='0.0.1') + #parser, opts, args = parseOpts(argv) # Open appropriate CookieJar if opts.cookiefile is None: From 90f577f4e9d7eb64b97209501ffbd954dd68a366 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 20:52:08 -0400 Subject: [PATCH 02/35] dropping in local copy of the docopt module --- youtube_dl/docopt.py | 582 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 582 insertions(+) create mode 100644 youtube_dl/docopt.py diff --git a/youtube_dl/docopt.py b/youtube_dl/docopt.py new file mode 100644 index 000000000..d39916781 --- /dev/null +++ b/youtube_dl/docopt.py @@ -0,0 +1,582 @@ +# -*- coding: utf-8 -*- +"""Pythonic command-line interface parser that will make you smile. + + * http://docopt.org + * Repository and issue-tracker: https://github.com/docopt/docopt + * Licensed under terms of MIT license (see LICENSE-MIT) + * Copyright (c) 2013 Vladimir Keleshev, vladimir@keleshev.com + +""" +import sys +import re + + +__all__ = ['docopt'] +__version__ = '0.6.1' + + +class DocoptLanguageError(Exception): + + """Error in construction of usage-message by developer.""" + + +class DocoptExit(SystemExit): + + """Exit in case user invoked program with incorrect arguments.""" + + usage = '' + + def __init__(self, message=''): + SystemExit.__init__(self, (message + '\n' + self.usage).strip()) + + +class Pattern(object): + + def __eq__(self, other): + return repr(self) == repr(other) + + def __hash__(self): + return hash(repr(self)) + + def fix(self): + self.fix_identities() + self.fix_repeating_arguments() + return self + + def fix_identities(self, uniq=None): + """Make pattern-tree tips point to same object if they are equal.""" + if not hasattr(self, 'children'): + return self + uniq = list(set(self.flat())) if uniq is None else uniq + for i, child in enumerate(self.children): + if not hasattr(child, 'children'): + assert child in uniq + self.children[i] = uniq[uniq.index(child)] + else: + child.fix_identities(uniq) + + def fix_repeating_arguments(self): + """Fix elements that should accumulate/increment values.""" + either = [list(child.children) for child in transform(self).children] + for case in either: + for e in [child for child in case if case.count(child) > 1]: + if type(e) is Argument or type(e) is Option and e.argcount: + if e.value is None: + e.value = [] + elif type(e.value) is not list: + e.value = e.value.split() + if type(e) is Command or type(e) is Option and e.argcount == 0: + e.value = 0 + return self + + +def transform(pattern): + """Expand pattern into an (almost) equivalent one, but with single Either. + + Example: ((-a | -b) (-c | -d)) => (-a -c | -a -d | -b -c | -b -d) + Quirks: [-a] => (-a), (-a...) => (-a -a) + + """ + result = [] + groups = [[pattern]] + while groups: + children = groups.pop(0) + parents = [Required, Optional, OptionsShortcut, Either, OneOrMore] + if any(t in map(type, children) for t in parents): + child = [c for c in children if type(c) in parents][0] + children.remove(child) + if type(child) is Either: + for c in child.children: + groups.append([c] + children) + elif type(child) is OneOrMore: + groups.append(child.children * 2 + children) + else: + groups.append(child.children + children) + else: + result.append(children) + return Either(*[Required(*e) for e in result]) + + +class LeafPattern(Pattern): + + """Leaf/terminal node of a pattern tree.""" + + def __init__(self, name, value=None): + self.name, self.value = name, value + + def __repr__(self): + return '%s(%r, %r)' % (self.__class__.__name__, self.name, self.value) + + def flat(self, *types): + return [self] if not types or type(self) in types else [] + + def match(self, left, collected=None): + collected = [] if collected is None else collected + pos, match = self.single_match(left) + if match is None: + return False, left, collected + left_ = left[:pos] + left[pos + 1:] + same_name = [a for a in collected if a.name == self.name] + if type(self.value) in (int, list): + if type(self.value) is int: + increment = 1 + else: + increment = ([match.value] if type(match.value) is str + else match.value) + if not same_name: + match.value = increment + return True, left_, collected + [match] + same_name[0].value += increment + return True, left_, collected + return True, left_, collected + [match] + + +class BranchPattern(Pattern): + + """Branch/inner node of a pattern tree.""" + + def __init__(self, *children): + self.children = list(children) + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, + ', '.join(repr(a) for a in self.children)) + + def flat(self, *types): + if type(self) in types: + return [self] + return sum([child.flat(*types) for child in self.children], []) + + +class Argument(LeafPattern): + + def single_match(self, left): + for n, pattern in enumerate(left): + if type(pattern) is Argument: + return n, Argument(self.name, pattern.value) + return None, None + + @classmethod + def parse(class_, source): + name = re.findall('(<\S*?>)', source)[0] + value = re.findall('\[default: (.*)\]', source, flags=re.I) + return class_(name, value[0] if value else None) + + +class Command(Argument): + + def __init__(self, name, value=False): + self.name, self.value = name, value + + def single_match(self, left): + for n, pattern in enumerate(left): + if type(pattern) is Argument: + if pattern.value == self.name: + return n, Command(self.name, True) + else: + break + return None, None + + +class Option(LeafPattern): + + def __init__(self, short=None, long=None, argcount=0, value=False): + assert argcount in (0, 1) + self.short, self.long, self.argcount = short, long, argcount + self.value = None if value is False and argcount else value + + @classmethod + def parse(class_, option_description): + short, long, argcount, value = None, None, 0, False + options, _, description = option_description.strip().partition(' ') + options = options.replace(',', ' ').replace('=', ' ') + for s in options.split(): + if s.startswith('--'): + long = s + elif s.startswith('-'): + short = s + else: + argcount = 1 + if argcount: + matched = re.findall('\[default: (.*)\]', description, flags=re.I) + value = matched[0] if matched else None + return class_(short, long, argcount, value) + + def single_match(self, left): + for n, pattern in enumerate(left): + if self.name == pattern.name: + return n, pattern + return None, None + + @property + def name(self): + return self.long or self.short + + def __repr__(self): + return 'Option(%r, %r, %r, %r)' % (self.short, self.long, + self.argcount, self.value) + + +class Required(BranchPattern): + + def match(self, left, collected=None): + collected = [] if collected is None else collected + l = left + c = collected + for pattern in self.children: + matched, l, c = pattern.match(l, c) + if not matched: + return False, left, collected + return True, l, c + + +class Optional(BranchPattern): + + def match(self, left, collected=None): + collected = [] if collected is None else collected + for pattern in self.children: + m, left, collected = pattern.match(left, collected) + return True, left, collected + + +class OptionsShortcut(Optional): + + """Marker/placeholder for [options] shortcut.""" + + +class OneOrMore(BranchPattern): + + def match(self, left, collected=None): + assert len(self.children) == 1 + collected = [] if collected is None else collected + l = left + c = collected + l_ = None + matched = True + times = 0 + while matched: + # could it be that something didn't match but changed l or c? + matched, l, c = self.children[0].match(l, c) + times += 1 if matched else 0 + if l_ == l: + break + l_ = l + if times >= 1: + return True, l, c + return False, left, collected + + +class Either(BranchPattern): + + def match(self, left, collected=None): + collected = [] if collected is None else collected + outcomes = [] + for pattern in self.children: + matched, _, _ = outcome = pattern.match(left, collected) + if matched: + outcomes.append(outcome) + if outcomes: + return min(outcomes, key=lambda outcome: len(outcome[1])) + return False, left, collected + + +class Tokens(list): + + def __init__(self, source, error=DocoptExit): + self += source.split() if hasattr(source, 'split') else source + self.error = error + + @staticmethod + def from_pattern(source): + source = re.sub(r'([\[\]\(\)\|]|\.\.\.)', r' \1 ', source) + source = [s for s in re.split('\s+|(\S*<.*?>)', source) if s] + return Tokens(source, error=DocoptLanguageError) + + def move(self): + return self.pop(0) if len(self) else None + + def current(self): + return self[0] if len(self) else None + + +def parse_long(tokens, options): + """long ::= '--' chars [ ( ' ' | '=' ) chars ] ;""" + long, eq, value = tokens.move().partition('=') + assert long.startswith('--') + value = None if eq == value == '' else value + similar = [o for o in options if o.long == long] + if tokens.error is DocoptExit and similar == []: # if no exact match + similar = [o for o in options if o.long and o.long.startswith(long)] + if len(similar) > 1: # might be simply specified ambiguously 2+ times? + raise tokens.error('%s is not a unique prefix: %s?' % + (long, ', '.join(o.long for o in similar))) + elif len(similar) < 1: + argcount = 1 if eq == '=' else 0 + o = Option(None, long, argcount) + options.append(o) + if tokens.error is DocoptExit: + o = Option(None, long, argcount, value if argcount else True) + else: + o = Option(similar[0].short, similar[0].long, + similar[0].argcount, similar[0].value) + if o.argcount == 0: + if value is not None: + raise tokens.error('%s must not have an argument' % o.long) + else: + if value is None: + if tokens.current() in [None, '--']: + raise tokens.error('%s requires argument' % o.long) + value = tokens.move() + if tokens.error is DocoptExit: + o.value = value if value is not None else True + return [o] + + +def parse_shorts(tokens, options): + """shorts ::= '-' ( chars )* [ [ ' ' ] chars ] ;""" + token = tokens.move() + assert token.startswith('-') and not token.startswith('--') + left = token.lstrip('-') + parsed = [] + while left != '': + short, left = '-' + left[0], left[1:] + similar = [o for o in options if o.short == short] + if len(similar) > 1: + raise tokens.error('%s is specified ambiguously %d times' % + (short, len(similar))) + elif len(similar) < 1: + o = Option(short, None, 0) + options.append(o) + if tokens.error is DocoptExit: + o = Option(short, None, 0, True) + else: # why copying is necessary here? + o = Option(short, similar[0].long, + similar[0].argcount, similar[0].value) + value = None + if o.argcount != 0: + if left == '': + if tokens.current() in [None, '--']: + raise tokens.error('%s requires argument' % short) + value = tokens.move() + else: + value = left + left = '' + if tokens.error is DocoptExit: + o.value = value if value is not None else True + parsed.append(o) + return parsed + + +def parse_pattern(source, options): + tokens = Tokens.from_pattern(source) + result = parse_expr(tokens, options) + if tokens.current() is not None: + raise tokens.error('unexpected ending: %r' % ' '.join(tokens)) + return Required(*result) + + +def parse_expr(tokens, options): + """expr ::= seq ( '|' seq )* ;""" + seq = parse_seq(tokens, options) + if tokens.current() != '|': + return seq + result = [Required(*seq)] if len(seq) > 1 else seq + while tokens.current() == '|': + tokens.move() + seq = parse_seq(tokens, options) + result += [Required(*seq)] if len(seq) > 1 else seq + return [Either(*result)] if len(result) > 1 else result + + +def parse_seq(tokens, options): + """seq ::= ( atom [ '...' ] )* ;""" + result = [] + while tokens.current() not in [None, ']', ')', '|']: + atom = parse_atom(tokens, options) + if tokens.current() == '...': + atom = [OneOrMore(*atom)] + tokens.move() + result += atom + return result + + +def parse_atom(tokens, options): + """atom ::= '(' expr ')' | '[' expr ']' | 'options' + | long | shorts | argument | command ; + """ + token = tokens.current() + result = [] + if token in '([': + tokens.move() + matching, pattern = {'(': [')', Required], '[': [']', Optional]}[token] + result = pattern(*parse_expr(tokens, options)) + if tokens.move() != matching: + raise tokens.error("unmatched '%s'" % token) + return [result] + elif token == 'options': + tokens.move() + return [OptionsShortcut()] + elif token.startswith('--') and token != '--': + return parse_long(tokens, options) + elif token.startswith('-') and token not in ('-', '--'): + return parse_shorts(tokens, options) + elif token.startswith('<') and token.endswith('>') or token.isupper(): + return [Argument(tokens.move())] + else: + return [Command(tokens.move())] + + +def parse_argv(tokens, options, options_first=False): + """Parse command-line argument vector. + + If options_first: + argv ::= [ long | shorts ]* [ argument ]* [ '--' [ argument ]* ] ; + else: + argv ::= [ long | shorts | argument ]* [ '--' [ argument ]* ] ; + + """ + parsed = [] + while tokens.current() is not None: + if tokens.current() == '--': + return parsed + [Argument(None, v) for v in tokens] + elif tokens.current().startswith('--'): + parsed += parse_long(tokens, options) + elif tokens.current().startswith('-') and tokens.current() != '-': + parsed += parse_shorts(tokens, options) + elif options_first: + return parsed + [Argument(None, v) for v in tokens] + else: + parsed.append(Argument(None, tokens.move())) + return parsed + + +def parse_defaults(doc): + defaults = [] + for s in parse_section('options:', doc): + # FIXME corner case "bla: options: --foo" + _, _, s = s.partition(':') # get rid of "options:" + split = re.split('\n[ \t]*(-\S+?)', '\n' + s)[1:] + split = [s1 + s2 for s1, s2 in zip(split[::2], split[1::2])] + options = [Option.parse(s) for s in split if s.startswith('-')] + defaults += options + return defaults + + +def parse_section(name, source): + pattern = re.compile('^([^\n]*' + name + '[^\n]*\n?(?:[ \t].*?(?:\n|$))*)', + re.IGNORECASE | re.MULTILINE) + return [s.strip() for s in pattern.findall(source)] + + +def formal_usage(section): + _, _, section = section.partition(':') # drop "usage:" + pu = section.split() + return '( ' + ' '.join(') | (' if s == pu[0] else s for s in pu[1:]) + ' )' + + +def extras(help, version, options, doc): + if help and any((o.name in ('-h', '--help')) and o.value for o in options): + print(doc.strip("\n")) + sys.exit() + if version and any(o.name == '--version' and o.value for o in options): + print(version) + sys.exit() + + +class Dict(dict): + def __repr__(self): + return '{%s}' % ',\n '.join('%r: %r' % i for i in sorted(self.items())) + + +def docopt(doc, argv=None, help=True, version=None, options_first=False): + """Parse `argv` based on command-line interface described in `doc`. + + `docopt` creates your command-line interface based on its + description that you pass as `doc`. Such description can contain + --options, , commands, which could be + [optional], (required), (mutually | exclusive) or repeated... + + Parameters + ---------- + doc : str + Description of your command-line interface. + argv : list of str, optional + Argument vector to be parsed. sys.argv[1:] is used if not + provided. + help : bool (default: True) + Set to False to disable automatic help on -h or --help + options. + version : any object + If passed, the object will be printed if --version is in + `argv`. + options_first : bool (default: False) + Set to True to require options precede positional arguments, + i.e. to forbid options and positional arguments intermix. + + Returns + ------- + args : dict + A dictionary, where keys are names of command-line elements + such as e.g. "--verbose" and "", and values are the + parsed values of those elements. + + Example + ------- + >>> from docopt import docopt + >>> doc = ''' + ... Usage: + ... my_program tcp [--timeout=] + ... my_program serial [--baud=] [--timeout=] + ... my_program (-h | --help | --version) + ... + ... Options: + ... -h, --help Show this screen and exit. + ... --baud= Baudrate [default: 9600] + ... ''' + >>> argv = ['tcp', '127.0.0.1', '80', '--timeout', '30'] + >>> docopt(doc, argv) + {'--baud': '9600', + '--help': False, + '--timeout': '30', + '--version': False, + '': '127.0.0.1', + '': '80', + 'serial': False, + 'tcp': True} + + See also + -------- + * For video introduction see http://docopt.org + * Full documentation is available in README.rst as well as online + at https://github.com/docopt/docopt#readme + + """ + argv = sys.argv[1:] if argv is None else argv + + usage_sections = parse_section('usage:', doc) + if len(usage_sections) == 0: + raise DocoptLanguageError('"usage:" (case-insensitive) not found.') + if len(usage_sections) > 1: + raise DocoptLanguageError('More than one "usage:" (case-insensitive).') + DocoptExit.usage = usage_sections[0] + + options = parse_defaults(doc) + pattern = parse_pattern(formal_usage(DocoptExit.usage), options) + # [default] syntax for argument is disabled + #for a in pattern.flat(Argument): + # same_name = [d for d in arguments if d.name == a.name] + # if same_name: + # a.value = same_name[0].value + argv = parse_argv(Tokens(argv), list(options), options_first) + pattern_options = set(pattern.flat(Option)) + for options_shortcut in pattern.flat(OptionsShortcut): + doc_options = parse_defaults(doc) + options_shortcut.children = list(set(doc_options) - pattern_options) + #if any_options: + # options_shortcut.children += [Option(o.short, o.long, o.argcount) + # for o in argv if type(o) is Option] + extras(help, version, argv, doc) + matched, left, collected = pattern.fix().match(argv) + if matched and left == []: # better error message if left? + return Dict((a.name, a.value) for a in (pattern.flat() + collected)) + raise DocoptExit() \ No newline at end of file From 93d739c7dd66fd584b95ad10d6064b187f24f367 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 20:55:40 -0400 Subject: [PATCH 03/35] converting opts.cookiefile to opts[--cookies] --- youtube_dl/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 6b0570973..7c97301c1 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -492,8 +492,8 @@ def parseOpts(overrideArguments=None): filesystem.add_option('--no-continue', action='store_false', dest='continue_dl', help='do not resume partially downloaded files (restart from beginning)') - filesystem.add_option('--cookies', - dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in') + #filesystem.add_option('--cookies', + # dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in') filesystem.add_option('--no-part', action='store_true', dest='nopart', help='do not use .part files', default=False) filesystem.add_option('--no-mtime', @@ -568,12 +568,12 @@ def _real_main(argv=None): #parser, opts, args = parseOpts(argv) # Open appropriate CookieJar - if opts.cookiefile is None: + if not opts['--cookies']: jar = compat_cookiejar.CookieJar() else: try: - jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile) - if os.access(opts.cookiefile, os.R_OK): + jar = compat_cookiejar.MozillaCookieJar(opts['--cookies']) + if os.access(opts['--cookies'], os.R_OK): jar.load() except (IOError, OSError) as err: if opts.verbose: @@ -838,7 +838,7 @@ def _real_main(argv=None): retcode = 101 # Dump cookie jar if requested - if opts.cookiefile is not None: + if opts['--cookies']: try: jar.save() except (IOError, OSError) as err: From 79ed9e9bac36782aa5e185b8a5e3d1e199c72469 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 21:00:48 -0400 Subject: [PATCH 04/35] converted --user-agent and --referer to docopts mode --- youtube_dl/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 7c97301c1..67264a264 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -338,10 +338,10 @@ def parseOpts(overrideArguments=None): general.add_option('--dump-user-agent', action='store_true', dest='dump_user_agent', help='display the current browser identification', default=False) - general.add_option('--user-agent', - dest='user_agent', help='specify a custom user agent', metavar='UA') - general.add_option('--referer', - dest='referer', help='specify a custom referer, use if the video access is restricted to one domain', + #general.add_option('--user-agent', + # dest='user_agent', help='specify a custom user agent', metavar='UA') + #general.add_option('--referer', + # dest='referer', help='specify a custom referer, use if the video access is restricted to one domain', metavar='REF', default=None) general.add_option('--list-extractors', action='store_true', dest='list_extractors', @@ -581,12 +581,12 @@ def _real_main(argv=None): sys.stderr.write(u'ERROR: unable to open cookie file\n') sys.exit(101) # Set user agent - if opts.user_agent is not None: - std_headers['User-Agent'] = opts.user_agent + if opts['--user-agent']: + std_headers['User-Agent'] = opts['--user-agent'] # Set referer - if opts.referer is not None: - std_headers['Referer'] = opts.referer + if opts['--referer']: + std_headers['Referer'] = opts['--referer'] # Dump user agent if opts.dump_user_agent: From 984a6b80f9bb1d59053b8c7084bbadf577913307 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 21:02:05 -0400 Subject: [PATCH 05/35] swapped my name from pablo to paul, since that's more... official --- youtube_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 67264a264..24174c6d8 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -214,7 +214,7 @@ __authors__ = ( 'Pierre Rudloff', 'Huarong Huo', 'Ismael Mejía', - 'Pablo Barton' + 'Paul Barton' ) __license__ = 'Public Domain' From 3dc8c8b19f42c91b2f5c554685022075f048f80e Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 21:13:24 -0400 Subject: [PATCH 06/35] Converted --proxy to docopt --- youtube_dl/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 24174c6d8..64398946b 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -349,7 +349,7 @@ def parseOpts(overrideArguments=None): general.add_option('--extractor-descriptions', action='store_true', dest='list_extractor_descriptions', help='Output descriptions of all supported extractors', default=False) - general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL') + #general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL') general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.') @@ -483,8 +483,8 @@ def parseOpts(overrideArguments=None): filesystem.add_option('--restrict-filenames', action='store_true', dest='restrictfilenames', help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False) - filesystem.add_option('-a', '--batch-file', - dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') + #filesystem.add_option('-a', '--batch-file', + # dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') filesystem.add_option('-w', '--no-overwrites', action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) filesystem.add_option('-c', '--continue', @@ -595,12 +595,12 @@ def _real_main(argv=None): # Batch file verification batchurls = [] - if opts.batchfile is not None: + if opts['--batch-file']: try: - if opts.batchfile == '-': + if opts['--batch-file'] == '-': batchfd = sys.stdin else: - batchfd = open(opts.batchfile, 'r') + batchfd = open(opts['--batch-file'], 'r') batchurls = batchfd.readlines() batchurls = [x.strip() for x in batchurls] batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)] @@ -613,11 +613,11 @@ def _real_main(argv=None): # General configuration cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar) - if opts.proxy is not None: - if opts.proxy == '': + if opts['--proxy']: + if opts['--proxy'] == '': # Will this ever happen? proxies = {} else: - proxies = {'http': opts.proxy, 'https': opts.proxy} + proxies = {'http': opts['--proxy'], 'https': opts['--proxy']} else: proxies = compat_urllib_request.getproxies() # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805) From 2f0dd2fac94d7449f29e9972c6b4c0b3cf1cca18 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 21:15:13 -0400 Subject: [PATCH 07/35] utils.make_HTTPS_handler now looks for the docopt version of --no-check-certificate --- youtube_dl/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 201802cee..3f7982cc7 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -487,9 +487,9 @@ def make_HTTPS_handler(opts): import ssl context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.set_default_verify_paths() - + context.verify_mode = (ssl.CERT_NONE - if opts.no_check_certificate + if opts['--no-check-certificate'] else ssl.CERT_REQUIRED) return compat_urllib_request.HTTPSHandler(context=context) @@ -707,7 +707,7 @@ def date_from_str(date_str): delta = datetime.timedelta(**{unit: time}) return today + delta return datetime.datetime.strptime(date_str, "%Y%m%d").date() - + class DateRange(object): """Represents a time interval between two dates""" def __init__(self, start=None, end=None): From 6184f65c064175fa6f9bb4734faad7373636f15a Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 21:16:36 -0400 Subject: [PATCH 08/35] commenting out the optparse --no-check-certificate --- youtube_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 64398946b..2c5916a45 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -350,7 +350,7 @@ def parseOpts(overrideArguments=None): action='store_true', dest='list_extractor_descriptions', help='Output descriptions of all supported extractors', default=False) #general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL') - general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.') + #general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.') selection.add_option('--playlist-start', From d2cc1a15d5cfb60502ac8bce9b17ac42302264f2 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 21:20:43 -0400 Subject: [PATCH 09/35] converted --list-extractors and --extractor-descriptions to docopt --- youtube_dl/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 2c5916a45..8ce83cd99 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -343,12 +343,12 @@ def parseOpts(overrideArguments=None): #general.add_option('--referer', # dest='referer', help='specify a custom referer, use if the video access is restricted to one domain', metavar='REF', default=None) - general.add_option('--list-extractors', - action='store_true', dest='list_extractors', - help='List all supported extractors and the URLs they would handle', default=False) - general.add_option('--extractor-descriptions', - action='store_true', dest='list_extractor_descriptions', - help='Output descriptions of all supported extractors', default=False) + #general.add_option('--list-extractors', + # action='store_true', dest='list_extractors', + # help='List all supported extractors and the URLs they would handle', default=False) + #general.add_option('--extractor-descriptions', + # action='store_true', dest='list_extractor_descriptions', + # help='Output descriptions of all supported extractors', default=False) #general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL') #general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.') @@ -635,7 +635,7 @@ def _real_main(argv=None): extractors = gen_extractors() - if opts.list_extractors: + if opts['--list-extractors']: for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()): compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) matchedUrls = [url for url in all_urls if ie.suitable(url)] @@ -643,7 +643,7 @@ def _real_main(argv=None): for mu in matchedUrls: compat_print(u' ' + mu) sys.exit(0) - if opts.list_extractor_descriptions: + if opts['--extractor-descriptions']: for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()): if not ie._WORKING: continue From 44be3dc22dd45c1fec544e2580916b214a9c404a Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 21:52:27 -0400 Subject: [PATCH 10/35] worked on converting --netrc, --username, and --password to docopt; added mutual exclusion to the usage rules so docopt should reject it --- youtube_dl/__init__.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 8ce83cd99..2635bcaff 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -24,7 +24,7 @@ Usage: [--all-formats] [--prefer-free-formats] [--max-quality FORMAT] [--list-formats] [--write-sub] [--write-auto-sub] [--all-subs] [--list-subs] [--sub-format FORMAT] [--sub-lang LANGS] - [--username USERNAME] [--password PASSWORD] [--netrc] + [--username USERNAME --password PASSWORD | --netrc] [--video-password PASSWORD] [--extract-audio] [--audio-format FORMAT] [--audio-quality QUALITY] [--recode-video FORMAT] [--keep-video] [--no-post-overwrites] @@ -184,7 +184,6 @@ Options: videos) """ - __authors__ = ( 'Ricardo Garcia Gonzalez', 'Danny Colligan', @@ -342,7 +341,7 @@ def parseOpts(overrideArguments=None): # dest='user_agent', help='specify a custom user agent', metavar='UA') #general.add_option('--referer', # dest='referer', help='specify a custom referer, use if the video access is restricted to one domain', - metavar='REF', default=None) + # metavar='REF', default=None) #general.add_option('--list-extractors', # action='store_true', dest='list_extractors', # help='List all supported extractors and the URLs they would handle', default=False) @@ -657,16 +656,16 @@ def _real_main(argv=None): # Conflicting, missing and erroneous options - if opts.usenetrc and (opts.username is not None or opts.password is not None): + if opts['--netrc'] and (opts['--username'] or opts['--password']): parser.error(u'using .netrc conflicts with giving username/password') - if opts.password is not None and opts.username is None: + if opts['--password'] and not opts['--username']: parser.error(u' account username missing\n') if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid): parser.error(u'using output template conflicts with using title, video ID or auto number') if opts.usetitle and opts.useid: parser.error(u'using title conflicts with using video ID') - if opts.username is not None and opts.password is None: - opts.password = getpass.getpass(u'Type account password and press return:') + if opts['--username'] and not opts['--password']: + opts['--password'] = getpass.getpass(u'Type account password and press return:') if opts.ratelimit is not None: numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) if numeric_limit is None: @@ -734,9 +733,9 @@ def _real_main(argv=None): # YoutubeDL ydl = YoutubeDL({ - 'usenetrc': opts.usenetrc, - 'username': opts.username, - 'password': opts.password, + 'usenetrc': opts['--netrc'], + 'username': opts['--username'], + 'password': opts['--password'], 'videopassword': opts.videopassword, 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat), 'forceurl': opts.geturl, From 05be634f5455cdc4aaff21448517f184693a6285 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 21:58:33 -0400 Subject: [PATCH 11/35] Commenting out code for netrc conflict with username and password, should no longer be necessary --- youtube_dl/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 2635bcaff..3d6b7c00a 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -656,8 +656,8 @@ def _real_main(argv=None): # Conflicting, missing and erroneous options - if opts['--netrc'] and (opts['--username'] or opts['--password']): - parser.error(u'using .netrc conflicts with giving username/password') + #if opts['--netrc'] and (opts['--username'] or opts['--password']): + # parser.error(u'using .netrc conflicts with giving username/password') if opts['--password'] and not opts['--username']: parser.error(u' account username missing\n') if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid): From 1ed905fbd1548e3e46f152528f22bf818636d94e Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 22:18:32 -0400 Subject: [PATCH 12/35] worked on video titling options, adding mutual exclusion rules and conversion for docopt, I almost certainly broke some code in here --- youtube_dl/__init__.py | 74 +++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 3d6b7c00a..615ad6fb3 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -12,7 +12,7 @@ Usage: [--max-filesize SIZE] [--date DATE] [--datebefore DATE] [--dateafter DATE] [--no-playlist] [--rate-limit LIMIT] [--retries RETRIES] [--buffer-size SIZE] [--no-resize-buffer] - [--title] [--id] [--literal] [--auto-number] [--output TEMPLATE] + [[--title --literal | --id] --auto-number | --output TEMPLATE] [--autonumber-size NUMBER] [--restrict-filenames] [--batch-file FILE] [--no-overwrites] [--continue] [--no-continue] [--cookies FILE] [--no-part] [--no-mtime] [--write-description] @@ -456,26 +456,26 @@ def parseOpts(overrideArguments=None): action='store_true', dest='dump_intermediate_pages', default=False, help='print downloaded pages to debug problems(very verbose)') - filesystem.add_option('-t', '--title', - action='store_true', dest='usetitle', help='use title in file name (default)', default=False) - filesystem.add_option('--id', - action='store_true', dest='useid', help='use only video ID in file name', default=False) - filesystem.add_option('-l', '--literal', - action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False) - filesystem.add_option('-A', '--auto-number', - action='store_true', dest='autonumber', - help='number downloaded files starting from 00000', default=False) - filesystem.add_option('-o', '--output', - dest='outtmpl', metavar='TEMPLATE', - help=('output filename template. Use %(title)s to get the title, ' - '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, ' - '%(autonumber)s to get an automatically incremented number, ' - '%(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), ' - '%(extractor)s for the provider (youtube, metacafe, etc), ' - '%(id)s for the video id , %(playlist)s for the playlist the video is in, ' - '%(playlist_index)s for the position in the playlist and %% for a literal percent. ' - 'Use - to output to stdout. Can also be used to download to a different directory, ' - 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')) + #filesystem.add_option('-t', '--title', + # action='store_true', dest='usetitle', help='use title in file name (default)', default=False) + #filesystem.add_option('--id', + # action='store_true', dest='useid', help='use only video ID in file name', default=False) + #filesystem.add_option('-l', '--literal', + # action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False) + #filesystem.add_option('-A', '--auto-number', + # action='store_true', dest='autonumber', + # help='number downloaded files starting from 00000', default=False) + #filesystem.add_option('-o', '--output', + # dest='outtmpl', metavar='TEMPLATE', + # help=('output filename template. Use %(title)s to get the title, ' + # '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, ' + # '%(autonumber)s to get an automatically incremented number, ' + # '%(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), ' + # '%(extractor)s for the provider (youtube, metacafe, etc), ' + # '%(id)s for the video id , %(playlist)s for the playlist the video is in, ' + # '%(playlist_index)s for the position in the playlist and %% for a literal percent. ' + # 'Use - to output to stdout. Can also be used to download to a different directory, ' + # 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')) filesystem.add_option('--autonumber-size', dest='autonumber_size', metavar='NUMBER', help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given') @@ -564,6 +564,10 @@ def _real_main(argv=None): codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) opts = docopt(__doc__, version='0.0.1') + #Support both --title and deprecated --literal + if not opts['--title']: + opts['--title'] = opts['--literal'] + #parser, opts, args = parseOpts(argv) # Open appropriate CookieJar @@ -660,12 +664,14 @@ def _real_main(argv=None): # parser.error(u'using .netrc conflicts with giving username/password') if opts['--password'] and not opts['--username']: parser.error(u' account username missing\n') - if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid): - parser.error(u'using output template conflicts with using title, video ID or auto number') - if opts.usetitle and opts.useid: - parser.error(u'using title conflicts with using video ID') if opts['--username'] and not opts['--password']: opts['--password'] = getpass.getpass(u'Type account password and press return:') + + #if opts['--output'] is not None and (opts['--title'] or opts['--auto-number'] or opts['--id']): + # parser.error(u'using output template conflicts with using title, video ID or auto number') + #if optsopts['--title'] and opts.useid: + # parser.error(u'using title conflicts with using video ID') + if opts.ratelimit is not None: numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) if numeric_limit is None: @@ -720,15 +726,15 @@ def _real_main(argv=None): if sys.version_info < (3,): # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems) - if opts.outtmpl is not None: - opts.outtmpl = opts.outtmpl.decode(preferredencoding()) - outtmpl =((opts.outtmpl is not None and opts.outtmpl) - or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s') + if opts['--output']: + opts['--output'] = opts['--output'].decode(preferredencoding()) + outtmpl =((opts['--output'] is not None and opts['--output']) + or (opts.format == '-1' and opts['--title'] and u'%(title)s-%(id)s-%(format)s.%(ext)s') or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s') - or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s') - or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s') - or (opts.useid and u'%(id)s.%(ext)s') - or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s') + or (opts['--title'] and opts['--auto-number'] and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s') + or (opts['--title'] and u'%(title)s-%(id)s.%(ext)s') + or (opts['--id'] and u'%(id)s.%(ext)s') + or (opts['--auto-number'] and u'%(autonumber)s-%(id)s.%(ext)s') or u'%(title)s-%(id)s.%(ext)s') # YoutubeDL @@ -764,7 +770,7 @@ def _real_main(argv=None): 'progress_with_newline': opts.progress_with_newline, 'playliststart': opts.playliststart, 'playlistend': opts.playlistend, - 'logtostderr': opts.outtmpl == '-', + 'logtostderr': opts['--output'] == '-', 'consoletitle': opts.consoletitle, 'nopart': opts.nopart, 'updatetime': opts.updatetime, From f2f43bee10701f333e4cfe4e838431c2470adedc Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 23:55:48 -0400 Subject: [PATCH 13/35] Converted all verbosity group options to docopt --- youtube_dl/__init__.py | 114 ++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 615ad6fb3..8628f9452 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -419,42 +419,42 @@ def parseOpts(overrideArguments=None): help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False) downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP) - verbosity.add_option('-q', '--quiet', - action='store_true', dest='quiet', help='activates quiet mode', default=False) - verbosity.add_option('-s', '--simulate', - action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False) - verbosity.add_option('--skip-download', - action='store_true', dest='skip_download', help='do not download the video', default=False) - verbosity.add_option('-g', '--get-url', - action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False) - verbosity.add_option('-e', '--get-title', - action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False) - verbosity.add_option('--get-id', - action='store_true', dest='getid', help='simulate, quiet but print id', default=False) - verbosity.add_option('--get-thumbnail', - action='store_true', dest='getthumbnail', - help='simulate, quiet but print thumbnail URL', default=False) - verbosity.add_option('--get-description', - action='store_true', dest='getdescription', - help='simulate, quiet but print video description', default=False) - verbosity.add_option('--get-filename', - action='store_true', dest='getfilename', - help='simulate, quiet but print output filename', default=False) - verbosity.add_option('--get-format', - action='store_true', dest='getformat', - help='simulate, quiet but print output format', default=False) - verbosity.add_option('--newline', - action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False) - verbosity.add_option('--no-progress', - action='store_true', dest='noprogress', help='do not print progress bar', default=False) - verbosity.add_option('--console-title', - action='store_true', dest='consoletitle', - help='display progress in console titlebar', default=False) - verbosity.add_option('-v', '--verbose', - action='store_true', dest='verbose', help='print various debugging information', default=False) - verbosity.add_option('--dump-intermediate-pages', - action='store_true', dest='dump_intermediate_pages', default=False, - help='print downloaded pages to debug problems(very verbose)') + #verbosity.add_option('-q', '--quiet', + # action='store_true', dest='quiet', help='activates quiet mode', default=False) + #verbosity.add_option('-s', '--simulate', + # action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False) + #verbosity.add_option('--skip-download', + # action='store_true', dest='skip_download', help='do not download the video', default=False) + #verbosity.add_option('-g', '--get-url', + # action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False) + #verbosity.add_option('-e', '--get-title', + # action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False) + #verbosity.add_option('--get-id', + # action='store_true', dest='getid', help='simulate, quiet but print id', default=False) + #verbosity.add_option('--get-thumbnail', + # action='store_true', dest='getthumbnail', + # help='simulate, quiet but print thumbnail URL', default=False) + #verbosity.add_option('--get-description', + # action='store_true', dest='getdescription', + # help='simulate, quiet but print video description', default=False) + #verbosity.add_option('--get-filename', + # action='store_true', dest='getfilename', + # help='simulate, quiet but print output filename', default=False) + #verbosity.add_option('--get-format', + # action='store_true', dest='getformat', + # help='simulate, quiet but print output format', default=False) + #verbosity.add_option('--newline', + # action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False) + #verbosity.add_option('--no-progress', + # action='store_true', dest='noprogress', help='do not print progress bar', default=False) + #verbosity.add_option('--console-title', + # action='store_true', dest='consoletitle', + # help='display progress in console titlebar', default=False) + #verbosity.add_option('-v', '--verbose', + # action='store_true', dest='verbose', help='print various debugging information', default=False) + #verbosity.add_option('--dump-intermediate-pages', + # action='store_true', dest='dump_intermediate_pages', default=False, + # help='print downloaded pages to debug problems(very verbose)') #filesystem.add_option('-t', '--title', # action='store_true', dest='usetitle', help='use title in file name (default)', default=False) @@ -537,7 +537,7 @@ def parseOpts(overrideArguments=None): if overrideArguments is not None: opts, args = parser.parse_args(overrideArguments) - if opts.verbose: + if opts['--verbose']: sys.stderr.write(u'[debug] Override config: ' + repr(overrideArguments) + '\n') else: xdg_config_home = os.environ.get('XDG_CONFIG_HOME') @@ -550,7 +550,7 @@ def parseOpts(overrideArguments=None): commandLineConf = sys.argv[1:] argv = systemConf + userConf + commandLineConf opts, args = parser.parse_args(argv) - if opts.verbose: + if opts['--verbose']: sys.stderr.write(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n') sys.stderr.write(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n') sys.stderr.write(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n') @@ -579,7 +579,7 @@ def _real_main(argv=None): if os.access(opts['--cookies'], os.R_OK): jar.load() except (IOError, OSError) as err: - if opts.verbose: + if opts['--verbose']: traceback.print_exc() sys.stderr.write(u'ERROR: unable to open cookie file\n') sys.exit(101) @@ -607,7 +607,7 @@ def _real_main(argv=None): batchurls = batchfd.readlines() batchurls = [x.strip() for x in batchurls] batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)] - if opts.verbose: + if opts['--verbose']: sys.stderr.write(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n') except IOError: sys.exit(u'ERROR: batch file could not be read') @@ -743,16 +743,16 @@ def _real_main(argv=None): 'username': opts['--username'], 'password': opts['--password'], 'videopassword': opts.videopassword, - 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat), - 'forceurl': opts.geturl, - 'forcetitle': opts.gettitle, - 'forceid': opts.getid, - 'forcethumbnail': opts.getthumbnail, - 'forcedescription': opts.getdescription, - 'forcefilename': opts.getfilename, - 'forceformat': opts.getformat, - 'simulate': opts.simulate, - 'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat), + 'quiet': (opts['--quiet'] or opts['--get-url'] or opts['--get-title'] or opts['--get-id'] or opts['--get-thumbnail'] or opts['--get-description'] or opts['--get-filename'] or opts['--get-format']), + 'forceurl': opts['--get-url'], + 'forcetitle': opts['--get-title'], + 'forceid': opts['--get-id'], + 'forcethumbnail': opts['--get-thumbnail'], + 'forcedescription': opts['--get-description'], + 'forcefilename': opts['--get-filename'], + 'forceformat': opts['--get-format'], + 'simulate': opts['--simulate'], + 'skip_download': (opts['--skip-download'] or opts['--simulate'] or opts['--get-url'] or opts['--get-title'] or opts['--get-id'] or opts['--get-thumbnail'] or opts['--get-description'] or opts['--get-filename'] or opts['--get-format']), 'format': opts.format, 'format_limit': opts.format_limit, 'listformats': opts.listformats, @@ -766,12 +766,12 @@ def _real_main(argv=None): 'buffersize': opts.buffersize, 'noresizebuffer': opts.noresizebuffer, 'continuedl': opts.continue_dl, - 'noprogress': opts.noprogress, - 'progress_with_newline': opts.progress_with_newline, + 'noprogress': opts['--no-progress'], + 'progress_with_newline': opts['--newline'], 'playliststart': opts.playliststart, 'playlistend': opts.playlistend, 'logtostderr': opts['--output'] == '-', - 'consoletitle': opts.consoletitle, + 'consoletitle': opts['--console-title'], 'nopart': opts.nopart, 'updatetime': opts.updatetime, 'writedescription': opts.writedescription, @@ -787,8 +787,8 @@ def _real_main(argv=None): 'rejecttitle': decodeOption(opts.rejecttitle), 'max_downloads': opts.max_downloads, 'prefer_free_formats': opts.prefer_free_formats, - 'verbose': opts.verbose, - 'dump_intermediate_pages': opts.dump_intermediate_pages, + 'verbose': opts['--verbose'], + 'dump_intermediate_pages': opts['--dump-intermediate-pages'], 'test': opts.test, 'keepvideo': opts.keepvideo, 'min_filesize': opts.min_filesize, @@ -796,7 +796,7 @@ def _real_main(argv=None): 'daterange': date, }) - if opts.verbose: + if opts['--verbose']: sys.stderr.write(u'[debug] youtube-dl version ' + __version__ + u'\n') try: sp = subprocess.Popen( @@ -827,7 +827,7 @@ def _real_main(argv=None): # Update version if opts.update_self: - update_self(ydl.to_screen, opts.verbose, sys.argv[0]) + update_self(ydl.to_screen, opts['--verbose'], sys.argv[0]) # Maybe do nothing if len(all_urls) < 1: From 273bf9bce618997fcee55fc64970d2eba63bb62f Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sat, 5 Oct 2013 23:59:34 -0400 Subject: [PATCH 14/35] Completed conversion of authentication options to docopt, commented out --- youtube_dl/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 8628f9452..cc63f0750 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -366,12 +366,12 @@ def parseOpts(overrideArguments=None): selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None) - authentication.add_option('-u', '--username', - dest='username', metavar='USERNAME', help='account username') - authentication.add_option('-p', '--password', - dest='password', metavar='PASSWORD', help='account password') - authentication.add_option('-n', '--netrc', - action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) + #authentication.add_option('-u', '--username', + # dest='username', metavar='USERNAME', help='account username') + #authentication.add_option('-p', '--password', + # dest='password', metavar='PASSWORD', help='account password') + #authentication.add_option('-n', '--netrc', + # action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) authentication.add_option('--video-password', dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)') @@ -742,7 +742,7 @@ def _real_main(argv=None): 'usenetrc': opts['--netrc'], 'username': opts['--username'], 'password': opts['--password'], - 'videopassword': opts.videopassword, + 'videopassword': opts['--video-password'], 'quiet': (opts['--quiet'] or opts['--get-url'] or opts['--get-title'] or opts['--get-id'] or opts['--get-thumbnail'] or opts['--get-description'] or opts['--get-filename'] or opts['--get-format']), 'forceurl': opts['--get-url'], 'forcetitle': opts['--get-title'], From cb1d0c4dc25aea36b719f780122f875819177be5 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 00:06:57 -0400 Subject: [PATCH 15/35] Converted --rate-limit and --min-filesize to docopt --- youtube_dl/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index cc63f0750..2f8b9c1d6 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -408,8 +408,8 @@ def parseOpts(overrideArguments=None): default=[], callback=_comma_separated_values_options_callback, help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'') - downloader.add_option('-r', '--rate-limit', - dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)') + #downloader.add_option('-r', '--rate-limit', + # dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)') downloader.add_option('-R', '--retries', dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10) downloader.add_option('--buffer-size', @@ -672,16 +672,16 @@ def _real_main(argv=None): #if optsopts['--title'] and opts.useid: # parser.error(u'using title conflicts with using video ID') - if opts.ratelimit is not None: - numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) + if opts['--rate-limit']: + numeric_limit = FileDownloader.parse_bytes(opts['--rate-limit']) if numeric_limit is None: parser.error(u'invalid rate limit specified') - opts.ratelimit = numeric_limit - if opts.min_filesize is not None: - numeric_limit = FileDownloader.parse_bytes(opts.min_filesize) + opts['--rate-limit'] = numeric_limit + if opts['--min-filesize']: + numeric_limit = FileDownloader.parse_bytes(opts['--min-filesize']) if numeric_limit is None: parser.error(u'invalid min_filesize specified') - opts.min_filesize = numeric_limit + opts['--min-filesize'] = numeric_limit if opts.max_filesize is not None: numeric_limit = FileDownloader.parse_bytes(opts.max_filesize) if numeric_limit is None: @@ -760,7 +760,7 @@ def _real_main(argv=None): 'autonumber_size': opts.autonumber_size, 'restrictfilenames': opts.restrictfilenames, 'ignoreerrors': opts.ignoreerrors, - 'ratelimit': opts.ratelimit, + 'ratelimit': opts['--rate-limit'], 'nooverwrites': opts.nooverwrites, 'retries': opts.retries, 'buffersize': opts.buffersize, @@ -791,7 +791,7 @@ def _real_main(argv=None): 'dump_intermediate_pages': opts['--dump-intermediate-pages'], 'test': opts.test, 'keepvideo': opts.keepvideo, - 'min_filesize': opts.min_filesize, + 'min_filesize': opts['--min-filesize'], 'max_filesize': opts.max_filesize, 'daterange': date, }) From 096f816c6ccf454fb7e480d85c263175276fc678 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 00:21:56 -0400 Subject: [PATCH 16/35] completed conversion of selection options to docopt --- youtube_dl/__init__.py | 63 +++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 2f8b9c1d6..671c5dec7 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -57,7 +57,8 @@ Options: Video Selection: --playlist-start NUMBER playlist video to start at [default: 1] - --playlist-end NUMBER playlist video to end at (default is last) + --playlist-end NUMBER playlist video to end at (defaults to last) + [default: -1] --match-title REGEX download only matching titles (regex or caseless sub-string) --reject-title REGEX skip download for matching titles (regex or @@ -352,18 +353,18 @@ def parseOpts(overrideArguments=None): #general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.') - selection.add_option('--playlist-start', - dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1) - selection.add_option('--playlist-end', - dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) - selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)') - selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)') - selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None) - selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None) - selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None) - selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None) - selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None) - selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None) + #selection.add_option('--playlist-start', + # dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1) + #selection.add_option('--playlist-end', + # dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) + #selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)') + #selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)') + #selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None) + #selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None) + #selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None) + #selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None) + #selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None) + #selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None) #authentication.add_option('-u', '--username', @@ -372,8 +373,8 @@ def parseOpts(overrideArguments=None): # dest='password', metavar='PASSWORD', help='account password') #authentication.add_option('-n', '--netrc', # action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) - authentication.add_option('--video-password', - dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)') + #authentication.add_option('--video-password', + # dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)') video_format.add_option('-f', '--format', @@ -682,11 +683,11 @@ def _real_main(argv=None): if numeric_limit is None: parser.error(u'invalid min_filesize specified') opts['--min-filesize'] = numeric_limit - if opts.max_filesize is not None: - numeric_limit = FileDownloader.parse_bytes(opts.max_filesize) + if opts['--max-filesize']: + numeric_limit = FileDownloader.parse_bytes(opts['--max-filesize']) if numeric_limit is None: parser.error(u'invalid max_filesize specified') - opts.max_filesize = numeric_limit + opts['--max-filesize'] = numeric_limit if opts.retries is not None: try: opts.retries = int(opts.retries) @@ -698,14 +699,14 @@ def _real_main(argv=None): parser.error(u'invalid buffer size specified') opts.buffersize = numeric_buffersize try: - opts.playliststart = int(opts.playliststart) - if opts.playliststart <= 0: + opts['--playlist-start'] = int(opts['--playlist-start']) + if opts['--playlist-start'] <= 0: raise ValueError(u'Playlist start must be positive') except (TypeError, ValueError) as err: parser.error(u'invalid playlist start number specified') try: - opts.playlistend = int(opts.playlistend) - if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart): + opts['--playlist-end'] = int(opts['--playlist-end']) + if opts['--playlist-end'] != -1 and (opts['--playlist-end'] <= 0 or opts['--playlist-end'] < opts['--playlist-start']): raise ValueError(u'Playlist end must be greater than playlist start') except (TypeError, ValueError) as err: parser.error(u'invalid playlist end number specified') @@ -719,10 +720,10 @@ def _real_main(argv=None): if opts.recodevideo is not None: if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']: parser.error(u'invalid video recode format specified') - if opts.date is not None: - date = DateRange.day(opts.date) + if opts['--date']: + date = DateRange.day(opts['--date']) else: - date = DateRange(opts.dateafter, opts.datebefore) + date = DateRange(opts['--dateafter'], opts['--datebefore']) if sys.version_info < (3,): # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems) @@ -768,8 +769,8 @@ def _real_main(argv=None): 'continuedl': opts.continue_dl, 'noprogress': opts['--no-progress'], 'progress_with_newline': opts['--newline'], - 'playliststart': opts.playliststart, - 'playlistend': opts.playlistend, + 'playliststart': opts['--playlist-start'], + 'playlistend': opts['--playlist-end'], 'logtostderr': opts['--output'] == '-', 'consoletitle': opts['--console-title'], 'nopart': opts.nopart, @@ -783,16 +784,16 @@ def _real_main(argv=None): 'listsubtitles': opts.listsubtitles, 'subtitlesformat': opts.subtitlesformat, 'subtitleslangs': opts.subtitleslangs, - 'matchtitle': decodeOption(opts.matchtitle), - 'rejecttitle': decodeOption(opts.rejecttitle), - 'max_downloads': opts.max_downloads, + 'matchtitle': decodeOption(opts['--match-title']), + 'rejecttitle': decodeOption(opts['--reject-title']), + 'max_downloads': opts['--max-downloads'], 'prefer_free_formats': opts.prefer_free_formats, 'verbose': opts['--verbose'], 'dump_intermediate_pages': opts['--dump-intermediate-pages'], 'test': opts.test, 'keepvideo': opts.keepvideo, 'min_filesize': opts['--min-filesize'], - 'max_filesize': opts.max_filesize, + 'max_filesize': opts['--max-filesize'], 'daterange': date, }) From 20779cfbae01ad5003775bdc344be761ed4a1e9f Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 00:28:12 -0400 Subject: [PATCH 17/35] completed conversion to docopt for all general options --- youtube_dl/__init__.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 671c5dec7..28a045341 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -327,17 +327,17 @@ def parseOpts(overrideArguments=None): filesystem = optparse.OptionGroup(parser, 'Filesystem Options') verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') - general.add_option('-h', '--help', - action='help', help='print this help text and exit') - general.add_option('-v', '--version', - action='version', help='print program version and exit') - general.add_option('-U', '--update', - action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)') - general.add_option('-i', '--ignore-errors', - action='store_true', dest='ignoreerrors', help='continue on download errors', default=False) - general.add_option('--dump-user-agent', - action='store_true', dest='dump_user_agent', - help='display the current browser identification', default=False) + #general.add_option('-h', '--help', + # action='help', help='print this help text and exit') + #general.add_option('-v', '--version', + # action='version', help='print program version and exit') + #general.add_option('-U', '--update', + # action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)') + #general.add_option('-i', '--ignore-errors', + # action='store_true', dest='ignoreerrors', help='continue on download errors', default=False) + #general.add_option('--dump-user-agent', + # action='store_true', dest='dump_user_agent', + # help='display the current browser identification', default=False) #general.add_option('--user-agent', # dest='user_agent', help='specify a custom user agent', metavar='UA') #general.add_option('--referer', @@ -593,7 +593,7 @@ def _real_main(argv=None): std_headers['Referer'] = opts['--referer'] # Dump user agent - if opts.dump_user_agent: + if opts['--dump-user-agent']: compat_print(std_headers['User-Agent']) sys.exit(0) @@ -760,7 +760,7 @@ def _real_main(argv=None): 'outtmpl': outtmpl, 'autonumber_size': opts.autonumber_size, 'restrictfilenames': opts.restrictfilenames, - 'ignoreerrors': opts.ignoreerrors, + 'ignoreerrors': opts['--ignore-errors'], 'ratelimit': opts['--rate-limit'], 'nooverwrites': opts.nooverwrites, 'retries': opts.retries, @@ -827,12 +827,12 @@ def _real_main(argv=None): ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat)) # Update version - if opts.update_self: + if opts['--update']: update_self(ydl.to_screen, opts['--verbose'], sys.argv[0]) # Maybe do nothing if len(all_urls) < 1: - if not opts.update_self: + if not opts['--update']: parser.error(u'you must provide at least one URL') else: sys.exit() From bee263ad72990161bb5422a122af61c1434f90c2 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 00:40:42 -0400 Subject: [PATCH 18/35] Converting --format, --all-formats, and --prefer-free-formats to docopt, --all-formats will override --format --- youtube_dl/__init__.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 28a045341..30030f1d4 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -377,13 +377,13 @@ def parseOpts(overrideArguments=None): # dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)') - video_format.add_option('-f', '--format', - action='store', dest='format', metavar='FORMAT', - help='video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported') - video_format.add_option('--all-formats', - action='store_const', dest='format', help='download all available video formats', const='all') - video_format.add_option('--prefer-free-formats', - action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested') + #video_format.add_option('-f', '--format', + # action='store', dest='format', metavar='FORMAT', + # help='video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported') + #video_format.add_option('--all-formats', + # action='store_const', dest='format', help='download all available video formats', const='all') + #video_format.add_option('--prefer-free-formats', + # action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested') video_format.add_option('--max-quality', action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') video_format.add_option('-F', '--list-formats', @@ -568,6 +568,9 @@ def _real_main(argv=None): #Support both --title and deprecated --literal if not opts['--title']: opts['--title'] = opts['--literal'] + #--all-formats will supersede --format + if opts['--all-formats']: + opts['--format'] = 'all' #parser, opts, args = parseOpts(argv) @@ -730,8 +733,8 @@ def _real_main(argv=None): if opts['--output']: opts['--output'] = opts['--output'].decode(preferredencoding()) outtmpl =((opts['--output'] is not None and opts['--output']) - or (opts.format == '-1' and opts['--title'] and u'%(title)s-%(id)s-%(format)s.%(ext)s') - or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s') + or (opts['--format'] == '-1' and opts['--title'] and u'%(title)s-%(id)s-%(format)s.%(ext)s') # Why would --format ever be -1 ? + or (opts['--format'] == '-1' and u'%(id)s-%(format)s.%(ext)s') # Why would --format ever be -1 ? or (opts['--title'] and opts['--auto-number'] and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s') or (opts['--title'] and u'%(title)s-%(id)s.%(ext)s') or (opts['--id'] and u'%(id)s.%(ext)s') @@ -754,7 +757,7 @@ def _real_main(argv=None): 'forceformat': opts['--get-format'], 'simulate': opts['--simulate'], 'skip_download': (opts['--skip-download'] or opts['--simulate'] or opts['--get-url'] or opts['--get-title'] or opts['--get-id'] or opts['--get-thumbnail'] or opts['--get-description'] or opts['--get-filename'] or opts['--get-format']), - 'format': opts.format, + 'format': opts['--format'], 'format_limit': opts.format_limit, 'listformats': opts.listformats, 'outtmpl': outtmpl, @@ -787,7 +790,7 @@ def _real_main(argv=None): 'matchtitle': decodeOption(opts['--match-title']), 'rejecttitle': decodeOption(opts['--reject-title']), 'max_downloads': opts['--max-downloads'], - 'prefer_free_formats': opts.prefer_free_formats, + 'prefer_free_formats': opts['--prefer-free-formats'], 'verbose': opts['--verbose'], 'dump_intermediate_pages': opts['--dump-intermediate-pages'], 'test': opts.test, From 452bce55a59dcb8a95f8a60ab94398b6b3f38007 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 00:41:25 -0400 Subject: [PATCH 19/35] Adding TODO note to remind me that I should look out for parameter checking that relies on None identity and not boolean --- youtube_dl/YoutubeDL.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index c2f992b8e..e78cb394a 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -16,6 +16,8 @@ from .utils import * from .extractor import get_info_extractor, gen_extractors from .FileDownloader import FileDownloader +#TODO: Check for parameters that eval None identity, docopt defaults to False + class YoutubeDL(object): """YoutubeDL class. @@ -80,7 +82,7 @@ class YoutubeDL(object): keepvideo: Keep the video file after post-processing daterange: A DateRange object, download only if the upload_date is in the range. skip_download: Skip the actual download of the video file - + The following parameters are not used by YoutubeDL itself, they are used by the FileDownloader: nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test, @@ -299,14 +301,14 @@ class YoutubeDL(object): if date not in dateRange: return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange) return None - + def extract_info(self, url, download=True, ie_key=None, extra_info={}): ''' Returns a list with a dictionary for each video we find. If 'download', also downloads the videos. extra_info is a dict containing the extra values to add to each result ''' - + if ie_key: ies = [self.get_info_extractor(ie_key)] else: @@ -348,7 +350,7 @@ class YoutubeDL(object): raise else: self.report_error(u'no suitable InfoExtractor: %s' % url) - + def process_ie_result(self, ie_result, download=True, extra_info={}): """ Take the result of the ie(may be modified) and resolve all unresolved @@ -399,7 +401,7 @@ class YoutubeDL(object): for i,entry in enumerate(entries,1): self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries)) extra = { - 'playlist': playlist, + 'playlist': playlist, 'playlist_index': i + playliststart, } if not 'extractor' in entry: From d297814621b64dd7086f72cf7802b274a2be2ac3 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 00:44:55 -0400 Subject: [PATCH 20/35] converting to docopt complete for video format options --- youtube_dl/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 30030f1d4..b61c780fc 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -384,10 +384,10 @@ def parseOpts(overrideArguments=None): # action='store_const', dest='format', help='download all available video formats', const='all') #video_format.add_option('--prefer-free-formats', # action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested') - video_format.add_option('--max-quality', - action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') - video_format.add_option('-F', '--list-formats', - action='store_true', dest='listformats', help='list all available formats (currently youtube only)') + #video_format.add_option('--max-quality', + # action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') + #video_format.add_option('-F', '--list-formats', + # action='store_true', dest='listformats', help='list all available formats (currently youtube only)') subtitles.add_option('--write-sub', '--write-srt', action='store_true', dest='writesubtitles', @@ -758,8 +758,8 @@ def _real_main(argv=None): 'simulate': opts['--simulate'], 'skip_download': (opts['--skip-download'] or opts['--simulate'] or opts['--get-url'] or opts['--get-title'] or opts['--get-id'] or opts['--get-thumbnail'] or opts['--get-description'] or opts['--get-filename'] or opts['--get-format']), 'format': opts['--format'], - 'format_limit': opts.format_limit, - 'listformats': opts.listformats, + 'format_limit': opts['--max-quality'], + 'listformats': opts['--list-formats'], 'outtmpl': outtmpl, 'autonumber_size': opts.autonumber_size, 'restrictfilenames': opts.restrictfilenames, From 6211b1a635135d26c029860beb550166bcab9ad3 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 01:03:15 -0400 Subject: [PATCH 21/35] Conversion done for subtitles options; secondary option names were dropped (only one option name per option now); multiple position arguments enabled for --sub-langs which may result in needed changes with URL, perhaps URL(s) should always be first --- youtube_dl/__init__.py | 66 +++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index b61c780fc..0a9a7bfec 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -23,13 +23,13 @@ Usage: [--verbose] [--dump-intermediate-pages] [--format FORMAT] [--all-formats] [--prefer-free-formats] [--max-quality FORMAT] [--list-formats] [--write-sub] [--write-auto-sub] [--all-subs] - [--list-subs] [--sub-format FORMAT] [--sub-lang LANGS] + [--list-subs] [--sub-format FORMAT] [--sub-lang LANGS ...] [--username USERNAME --password PASSWORD | --netrc] [--video-password PASSWORD] [--extract-audio] [--audio-format FORMAT] [--audio-quality QUALITY] [--recode-video FORMAT] [--keep-video] [--no-post-overwrites] [--embed-subs] - URL [url...] + URL [url ...] Options: General Options: @@ -155,10 +155,10 @@ Options: --all-subs downloads all the available subtitles of the video --list-subs lists all available subtitles for the video - --sub-format FORMAT subtitle format (default=srt) ([sbv/vtt] youtube - only) - --sub-lang LANGS languages of the subtitles to download (optional) - separated by commas, use IETF language tags like + --sub-format FORMAT subtitle format ([sbv/vtt] youtube only) + [default: srt] + --sub-lang LANGS ... languages of the subtitles to download (optional, + multiple arguments) use IETF language tags like 'en,pt' Authentication Options: @@ -185,7 +185,7 @@ Options: videos) """ -__authors__ = ( +__authors__ = ( 'Ricardo Garcia Gonzalez', 'Danny Colligan', 'Benjamin Johnson', @@ -389,25 +389,25 @@ def parseOpts(overrideArguments=None): #video_format.add_option('-F', '--list-formats', # action='store_true', dest='listformats', help='list all available formats (currently youtube only)') - subtitles.add_option('--write-sub', '--write-srt', - action='store_true', dest='writesubtitles', - help='write subtitle file', default=False) - subtitles.add_option('--write-auto-sub', '--write-automatic-sub', - action='store_true', dest='writeautomaticsub', - help='write automatic subtitle file (youtube only)', default=False) - subtitles.add_option('--all-subs', - action='store_true', dest='allsubtitles', - help='downloads all the available subtitles of the video', default=False) - subtitles.add_option('--list-subs', - action='store_true', dest='listsubtitles', - help='lists all available subtitles for the video', default=False) - subtitles.add_option('--sub-format', - action='store', dest='subtitlesformat', metavar='FORMAT', - help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt') - subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang', - action='callback', dest='subtitleslangs', metavar='LANGS', type='str', - default=[], callback=_comma_separated_values_options_callback, - help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'') + #subtitles.add_option('--write-sub', '--write-srt', + # action='store_true', dest='writesubtitles', + # help='write subtitle file', default=False) + #subtitles.add_option('--write-auto-sub', '--write-automatic-sub', + # action='store_true', dest='writeautomaticsub', + # help='write automatic subtitle file (youtube only)', default=False) + #subtitles.add_option('--all-subs', + # action='store_true', dest='allsubtitles', + # help='downloads all the available subtitles of the video', default=False) + #subtitles.add_option('--list-subs', + # action='store_true', dest='listsubtitles', + # help='lists all available subtitles for the video', default=False) + #subtitles.add_option('--sub-format', + # action='store', dest='subtitlesformat', metavar='FORMAT', + # help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt') + #subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang', + # action='callback', dest='subtitleslangs', metavar='LANGS', type='str', + # default=[], callback=_comma_separated_values_options_callback, + # help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'') #downloader.add_option('-r', '--rate-limit', # dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)') @@ -781,12 +781,12 @@ def _real_main(argv=None): 'writedescription': opts.writedescription, 'writeinfojson': opts.writeinfojson, 'writethumbnail': opts.writethumbnail, - 'writesubtitles': opts.writesubtitles, - 'writeautomaticsub': opts.writeautomaticsub, - 'allsubtitles': opts.allsubtitles, - 'listsubtitles': opts.listsubtitles, - 'subtitlesformat': opts.subtitlesformat, - 'subtitleslangs': opts.subtitleslangs, + 'writesubtitles': opts['--write-sub'], + 'writeautomaticsub': opts['--write-auto-sub'], + 'allsubtitles': opts['--all-subs'], + 'listsubtitles': opts['--list-subs'], + 'subtitlesformat': opts['--sub-format'], + 'subtitleslangs': opts['--sub-lang'], 'matchtitle': decodeOption(opts['--match-title']), 'rejecttitle': decodeOption(opts['--reject-title']), 'max_downloads': opts['--max-downloads'], @@ -827,7 +827,7 @@ def _real_main(argv=None): if opts.recodevideo: ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo)) if opts.embedsubtitles: - ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat)) + ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts['--sub-format'])) # Update version if opts['--update']: From b3a5c4a3cd11e8eec0defec9f6967aa16fc73fa7 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 09:37:41 -0400 Subject: [PATCH 22/35] completed conversion to docopt for the downloader options --- youtube_dl/__init__.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 0a9a7bfec..3caa2d35f 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -23,7 +23,7 @@ Usage: [--verbose] [--dump-intermediate-pages] [--format FORMAT] [--all-formats] [--prefer-free-formats] [--max-quality FORMAT] [--list-formats] [--write-sub] [--write-auto-sub] [--all-subs] - [--list-subs] [--sub-format FORMAT] [--sub-lang LANGS ...] + [--list-subs] [--sub-format FORMAT] [--sub-langs LANGS ...] [--username USERNAME --password PASSWORD | --netrc] [--video-password PASSWORD] [--extract-audio] [--audio-format FORMAT] [--audio-quality QUALITY] @@ -77,7 +77,7 @@ Options: -r, --rate-limit LIMIT maximum download rate (e.g. 50k or 44.6m) -R, --retries RETRIES number of retries [default: 10] --buffer-size SIZE size of download buffer (e.g. 1024 or 16k) - (default is 1024) + [default: 1024] --no-resize-buffer do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE. @@ -157,7 +157,7 @@ Options: --list-subs lists all available subtitles for the video --sub-format FORMAT subtitle format ([sbv/vtt] youtube only) [default: srt] - --sub-lang LANGS ... languages of the subtitles to download (optional, + --sub-langs LANGS ... languages of the subtitles to download (optional, multiple arguments) use IETF language tags like 'en,pt' @@ -411,14 +411,14 @@ def parseOpts(overrideArguments=None): #downloader.add_option('-r', '--rate-limit', # dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)') - downloader.add_option('-R', '--retries', - dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10) - downloader.add_option('--buffer-size', - dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024") - downloader.add_option('--no-resize-buffer', - action='store_true', dest='noresizebuffer', - help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False) - downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP) + #downloader.add_option('-R', '--retries', + # dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10) + #downloader.add_option('--buffer-size', + # dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024") + #downloader.add_option('--no-resize-buffer', + # action='store_true', dest='noresizebuffer', + # help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False) + #downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP) #verbosity.add_option('-q', '--quiet', # action='store_true', dest='quiet', help='activates quiet mode', default=False) @@ -691,16 +691,16 @@ def _real_main(argv=None): if numeric_limit is None: parser.error(u'invalid max_filesize specified') opts['--max-filesize'] = numeric_limit - if opts.retries is not None: + if opts['--retries'] is not None: # This should always be true, it has a default try: - opts.retries = int(opts.retries) + opts['--retries'] = int(opts['--retries']) except (TypeError, ValueError) as err: parser.error(u'invalid retry count specified') - if opts.buffersize is not None: - numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize) + if opts['--buffer-size'] is not None: # This should always be true, it has a default + numeric_buffersize = FileDownloader.parse_bytes(opts['--buffer-size']) if numeric_buffersize is None: parser.error(u'invalid buffer size specified') - opts.buffersize = numeric_buffersize + opts['--buffer-size'] = numeric_buffersize try: opts['--playlist-start'] = int(opts['--playlist-start']) if opts['--playlist-start'] <= 0: @@ -766,9 +766,9 @@ def _real_main(argv=None): 'ignoreerrors': opts['--ignore-errors'], 'ratelimit': opts['--rate-limit'], 'nooverwrites': opts.nooverwrites, - 'retries': opts.retries, - 'buffersize': opts.buffersize, - 'noresizebuffer': opts.noresizebuffer, + 'retries': opts['--retries'], + 'buffersize': opts['--buffer-size'], + 'noresizebuffer': opts['--no-resize-buffer'], 'continuedl': opts.continue_dl, 'noprogress': opts['--no-progress'], 'progress_with_newline': opts['--newline'], @@ -786,14 +786,14 @@ def _real_main(argv=None): 'allsubtitles': opts['--all-subs'], 'listsubtitles': opts['--list-subs'], 'subtitlesformat': opts['--sub-format'], - 'subtitleslangs': opts['--sub-lang'], + 'subtitleslangs': opts['--sub-langs'], 'matchtitle': decodeOption(opts['--match-title']), 'rejecttitle': decodeOption(opts['--reject-title']), 'max_downloads': opts['--max-downloads'], 'prefer_free_formats': opts['--prefer-free-formats'], 'verbose': opts['--verbose'], 'dump_intermediate_pages': opts['--dump-intermediate-pages'], - 'test': opts.test, + 'test': opts['--test'], 'keepvideo': opts.keepvideo, 'min_filesize': opts['--min-filesize'], 'max_filesize': opts['--max-filesize'], From e2ab108c2111f67c3354dd7e26f086e5f3114ee6 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 09:58:27 -0400 Subject: [PATCH 23/35] converted a few filesystem options to docopt, --continue and --no-continue have been made mutually exclusive and partial continuation will be skipped only if --no-continue is used --- youtube_dl/__init__.py | 48 ++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 3caa2d35f..1bf2015a4 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -14,7 +14,7 @@ Usage: [--retries RETRIES] [--buffer-size SIZE] [--no-resize-buffer] [[--title --literal | --id] --auto-number | --output TEMPLATE] [--autonumber-size NUMBER] [--restrict-filenames] - [--batch-file FILE] [--no-overwrites] [--continue] [--no-continue] + [--batch-file FILE] [--no-overwrites] [--continue | --no-continue] [--cookies FILE] [--no-part] [--no-mtime] [--write-description] [--write-info-json] [--write-thumbnail] [--quiet] [--simulate] [--skip-download] [--get-url] [--get-title] [--get-id] @@ -477,25 +477,25 @@ def parseOpts(overrideArguments=None): # '%(playlist_index)s for the position in the playlist and %% for a literal percent. ' # 'Use - to output to stdout. Can also be used to download to a different directory, ' # 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')) - filesystem.add_option('--autonumber-size', - dest='autonumber_size', metavar='NUMBER', - help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given') - filesystem.add_option('--restrict-filenames', - action='store_true', dest='restrictfilenames', - help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False) + #filesystem.add_option('--autonumber-size', + # dest='autonumber_size', metavar='NUMBER', + # help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given') + #filesystem.add_option('--restrict-filenames', + # action='store_true', dest='restrictfilenames', + # help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False) #filesystem.add_option('-a', '--batch-file', # dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') - filesystem.add_option('-w', '--no-overwrites', - action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) - filesystem.add_option('-c', '--continue', - action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True) - filesystem.add_option('--no-continue', - action='store_false', dest='continue_dl', - help='do not resume partially downloaded files (restart from beginning)') + #filesystem.add_option('-w', '--no-overwrites', + # action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) + #filesystem.add_option('-c', '--continue', + # action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True) + #filesystem.add_option('--no-continue', + # action='store_false', dest='continue_dl', + # help='do not resume partially downloaded files (restart from beginning)') #filesystem.add_option('--cookies', # dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in') - filesystem.add_option('--no-part', - action='store_true', dest='nopart', help='do not use .part files', default=False) + #filesystem.add_option('--no-part', + # action='store_true', dest='nopart', help='do not use .part files', default=False) filesystem.add_option('--no-mtime', action='store_false', dest='updatetime', help='do not use the Last-modified header to set the file modification time', default=True) @@ -571,6 +571,12 @@ def _real_main(argv=None): #--all-formats will supersede --format if opts['--all-formats']: opts['--format'] = 'all' + #--continue and --no-continue are mutually exclusive and control the + #partial downloading of files + if opts['--no-continue']: + opts['--continue'] = False + else: + opts['--continue'] = True #parser, opts, args = parseOpts(argv) @@ -761,22 +767,22 @@ def _real_main(argv=None): 'format_limit': opts['--max-quality'], 'listformats': opts['--list-formats'], 'outtmpl': outtmpl, - 'autonumber_size': opts.autonumber_size, - 'restrictfilenames': opts.restrictfilenames, + 'autonumber_size': opts['--autonumber-size'], + 'restrictfilenames': opts['--restrict-filenames'], 'ignoreerrors': opts['--ignore-errors'], 'ratelimit': opts['--rate-limit'], - 'nooverwrites': opts.nooverwrites, + 'nooverwrites': opts['--no-overwrites'], 'retries': opts['--retries'], 'buffersize': opts['--buffer-size'], 'noresizebuffer': opts['--no-resize-buffer'], - 'continuedl': opts.continue_dl, + 'continuedl': opts['--continue'], 'noprogress': opts['--no-progress'], 'progress_with_newline': opts['--newline'], 'playliststart': opts['--playlist-start'], 'playlistend': opts['--playlist-end'], 'logtostderr': opts['--output'] == '-', 'consoletitle': opts['--console-title'], - 'nopart': opts.nopart, + 'nopart': opts['--no-part'], 'updatetime': opts.updatetime, 'writedescription': opts.writedescription, 'writeinfojson': opts.writeinfojson, From dc7e144cdec2b84712bb4ae6994fc5eef00af033 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 10:03:56 -0400 Subject: [PATCH 24/35] --no-mtime now converted to docopt with proper boolean flipping --- youtube_dl/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 1bf2015a4..e12b5cda0 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -496,9 +496,9 @@ def parseOpts(overrideArguments=None): # dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in') #filesystem.add_option('--no-part', # action='store_true', dest='nopart', help='do not use .part files', default=False) - filesystem.add_option('--no-mtime', - action='store_false', dest='updatetime', - help='do not use the Last-modified header to set the file modification time', default=True) + #filesystem.add_option('--no-mtime', + # action='store_false', dest='updatetime', + # help='do not use the Last-modified header to set the file modification time', default=True) filesystem.add_option('--write-description', action='store_true', dest='writedescription', help='write video description to a .description file', default=False) @@ -577,6 +577,8 @@ def _real_main(argv=None): opts['--continue'] = False else: opts['--continue'] = True + #Invert boolean value of opts['--no-mtime'] + opts['--no-mtime'] = not opts['--no-mtime'] #parser, opts, args = parseOpts(argv) @@ -783,7 +785,7 @@ def _real_main(argv=None): 'logtostderr': opts['--output'] == '-', 'consoletitle': opts['--console-title'], 'nopart': opts['--no-part'], - 'updatetime': opts.updatetime, + 'updatetime': opts['--no-mtime'], 'writedescription': opts.writedescription, 'writeinfojson': opts.writeinfojson, 'writethumbnail': opts.writethumbnail, From 89c3485c4474f73149387456eb908f972a2781cb Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 10:06:45 -0400 Subject: [PATCH 25/35] filesystem options now all converted to docopt --- youtube_dl/__init__.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index e12b5cda0..56416a47f 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -499,15 +499,15 @@ def parseOpts(overrideArguments=None): #filesystem.add_option('--no-mtime', # action='store_false', dest='updatetime', # help='do not use the Last-modified header to set the file modification time', default=True) - filesystem.add_option('--write-description', - action='store_true', dest='writedescription', - help='write video description to a .description file', default=False) - filesystem.add_option('--write-info-json', - action='store_true', dest='writeinfojson', - help='write video metadata to a .info.json file', default=False) - filesystem.add_option('--write-thumbnail', - action='store_true', dest='writethumbnail', - help='write thumbnail image to disk', default=False) + #filesystem.add_option('--write-description', + # action='store_true', dest='writedescription', + # help='write video description to a .description file', default=False) + #filesystem.add_option('--write-info-json', + # action='store_true', dest='writeinfojson', + # help='write video metadata to a .info.json file', default=False) + #filesystem.add_option('--write-thumbnail', + # action='store_true', dest='writethumbnail', + # help='write thumbnail image to disk', default=False) postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False, @@ -786,9 +786,9 @@ def _real_main(argv=None): 'consoletitle': opts['--console-title'], 'nopart': opts['--no-part'], 'updatetime': opts['--no-mtime'], - 'writedescription': opts.writedescription, - 'writeinfojson': opts.writeinfojson, - 'writethumbnail': opts.writethumbnail, + 'writedescription': opts['--write-description'], + 'writeinfojson': opts['--write-info-json'], + 'writethumbnail': opts['--write-thumbnail'], 'writesubtitles': opts['--write-sub'], 'writeautomaticsub': opts['--write-auto-sub'], 'allsubtitles': opts['--all-subs'], From 48b37900b48c3a11218c8855f799b383580331c3 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 10:27:30 -0400 Subject: [PATCH 26/35] --extract-audio, --audio-format, --audio-quality, and --recode-video have been converted to docopt, only a few more postproc options remain to be converted --- youtube_dl/__init__.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 56416a47f..2fdd8319f 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -510,14 +510,14 @@ def parseOpts(overrideArguments=None): # help='write thumbnail image to disk', default=False) - postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False, - help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)') - postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', - help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default') - postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5', - help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)') - postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None, - help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)') + #postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False, + # help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)') + #postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', + # help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default') + #postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5', + # help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)') + #postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None, + # help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)') postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, help='keeps the video file on disk after the post-processing; the video is erased by default') postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False, @@ -721,15 +721,15 @@ def _real_main(argv=None): raise ValueError(u'Playlist end must be greater than playlist start') except (TypeError, ValueError) as err: parser.error(u'invalid playlist end number specified') - if opts.extractaudio: - if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']: + if opts['--extract-audio']: + if opts['--audio-format'] not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']: parser.error(u'invalid audio format specified') - if opts.audioquality: - opts.audioquality = opts.audioquality.strip('k').strip('K') - if not opts.audioquality.isdigit(): + if opts['--audio-quality']: + opts['--audio-quality'] = opts['--audio-quality'].strip('k').strip('K') + if not opts['--audio-quality'].isdigit(): parser.error(u'invalid audio quality specified') - if opts.recodevideo is not None: - if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']: + if opts['--recode-video']: + if opts['--recode-video'] not in ['mp4', 'flv', 'webm', 'ogg']: parser.error(u'invalid video recode format specified') if opts['--date']: date = DateRange.day(opts['--date']) @@ -830,10 +830,10 @@ def _real_main(argv=None): ydl.add_default_info_extractors() # PostProcessors - if opts.extractaudio: - ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites)) - if opts.recodevideo: - ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo)) + if opts['--extract-audio']: + ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts['--audio-format'], preferredquality=opts['--audio-quality'], nopostoverwrites=opts.nopostoverwrites)) + if opts['--recode-video']: + ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts['--recode-video'])) if opts.embedsubtitles: ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts['--sub-format'])) From eecd5c4adb40a8d18a8c0d55dbfddf3ac21a4fc1 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 10:41:44 -0400 Subject: [PATCH 27/35] The last postproc options have been converted to docopt --- youtube_dl/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 2fdd8319f..e4a26a1e8 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -518,10 +518,10 @@ def parseOpts(overrideArguments=None): # help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)') #postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None, # help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)') - postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, - help='keeps the video file on disk after the post-processing; the video is erased by default') - postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False, - help='do not overwrite post-processed files; the post-processed files are overwritten by default') + #postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, + # help='keeps the video file on disk after the post-processing; the video is erased by default') + #postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False, + # help='do not overwrite post-processed files; the post-processed files are overwritten by default') postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False, help='embed subtitles in the video (only for mp4 videos)') @@ -802,7 +802,7 @@ def _real_main(argv=None): 'verbose': opts['--verbose'], 'dump_intermediate_pages': opts['--dump-intermediate-pages'], 'test': opts['--test'], - 'keepvideo': opts.keepvideo, + 'keepvideo': opts['--keep-video'], 'min_filesize': opts['--min-filesize'], 'max_filesize': opts['--max-filesize'], 'daterange': date, @@ -831,10 +831,10 @@ def _real_main(argv=None): # PostProcessors if opts['--extract-audio']: - ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts['--audio-format'], preferredquality=opts['--audio-quality'], nopostoverwrites=opts.nopostoverwrites)) + ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts['--audio-format'], preferredquality=opts['--audio-quality'], nopostoverwrites=opts['--no-post-overwrites'])) if opts['--recode-video']: ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts['--recode-video'])) - if opts.embedsubtitles: + if opts['--embed-subs']: ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts['--sub-format'])) # Update version From 9d37d0375e9d077643f88738cd5110e81f797b87 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 13:31:44 -0400 Subject: [PATCH 28/35] swapping out parser.error() calls with custom OptionsError exceptions --- youtube_dl/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index e4a26a1e8..58303e0e7 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -241,6 +241,9 @@ from .extractor import gen_extractors from .YoutubeDL import YoutubeDL from .PostProcessor import * +class OptionsError(Exception): + pass + def parseOpts(overrideArguments=None): def _readOptions(filename_bytes): try: From c58edee57d9ffdeaa5375c04e91a6342c5ce1fd7 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 13:32:20 -0400 Subject: [PATCH 29/35] more swapping out parser.error() calls with custom OptionsError exceptions --- youtube_dl/__init__.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 58303e0e7..4d2c1e1b5 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -678,62 +678,62 @@ def _real_main(argv=None): #if opts['--netrc'] and (opts['--username'] or opts['--password']): # parser.error(u'using .netrc conflicts with giving username/password') if opts['--password'] and not opts['--username']: - parser.error(u' account username missing\n') + raise OptionsError(u' account username missing\n') if opts['--username'] and not opts['--password']: opts['--password'] = getpass.getpass(u'Type account password and press return:') #if opts['--output'] is not None and (opts['--title'] or opts['--auto-number'] or opts['--id']): - # parser.error(u'using output template conflicts with using title, video ID or auto number') + # raise OptionsError(u'using output template conflicts with using title, video ID or auto number') #if optsopts['--title'] and opts.useid: # parser.error(u'using title conflicts with using video ID') if opts['--rate-limit']: numeric_limit = FileDownloader.parse_bytes(opts['--rate-limit']) if numeric_limit is None: - parser.error(u'invalid rate limit specified') + raise OptionsError(u'invalid rate limit specified') opts['--rate-limit'] = numeric_limit if opts['--min-filesize']: numeric_limit = FileDownloader.parse_bytes(opts['--min-filesize']) if numeric_limit is None: - parser.error(u'invalid min_filesize specified') + raise OptionsError(u'invalid min_filesize specified') opts['--min-filesize'] = numeric_limit if opts['--max-filesize']: numeric_limit = FileDownloader.parse_bytes(opts['--max-filesize']) if numeric_limit is None: - parser.error(u'invalid max_filesize specified') + raise OptionsError(u'invalid max_filesize specified') opts['--max-filesize'] = numeric_limit if opts['--retries'] is not None: # This should always be true, it has a default try: opts['--retries'] = int(opts['--retries']) except (TypeError, ValueError) as err: - parser.error(u'invalid retry count specified') + raise OptionsError(u'invalid retry count specified') if opts['--buffer-size'] is not None: # This should always be true, it has a default numeric_buffersize = FileDownloader.parse_bytes(opts['--buffer-size']) if numeric_buffersize is None: - parser.error(u'invalid buffer size specified') + raise OptionsError(u'invalid buffer size specified') opts['--buffer-size'] = numeric_buffersize try: opts['--playlist-start'] = int(opts['--playlist-start']) if opts['--playlist-start'] <= 0: raise ValueError(u'Playlist start must be positive') except (TypeError, ValueError) as err: - parser.error(u'invalid playlist start number specified') + raise OptionsError(u'invalid playlist start number specified') try: opts['--playlist-end'] = int(opts['--playlist-end']) if opts['--playlist-end'] != -1 and (opts['--playlist-end'] <= 0 or opts['--playlist-end'] < opts['--playlist-start']): raise ValueError(u'Playlist end must be greater than playlist start') except (TypeError, ValueError) as err: - parser.error(u'invalid playlist end number specified') + raise OptionsError(u'invalid playlist end number specified') if opts['--extract-audio']: if opts['--audio-format'] not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']: - parser.error(u'invalid audio format specified') + raise OptionsError(u'invalid audio format specified') if opts['--audio-quality']: opts['--audio-quality'] = opts['--audio-quality'].strip('k').strip('K') if not opts['--audio-quality'].isdigit(): - parser.error(u'invalid audio quality specified') + raise OptionsError(u'invalid audio quality specified') if opts['--recode-video']: if opts['--recode-video'] not in ['mp4', 'flv', 'webm', 'ogg']: - parser.error(u'invalid video recode format specified') + raise OptionsError(u'invalid video recode format specified') if opts['--date']: date = DateRange.day(opts['--date']) else: @@ -847,7 +847,7 @@ def _real_main(argv=None): # Maybe do nothing if len(all_urls) < 1: if not opts['--update']: - parser.error(u'you must provide at least one URL') + raise OptionsError(u'you must provide at least one URL') else: sys.exit() From 24ccd1b862b61399012e6644c217d8243b78762b Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 16:48:59 -0400 Subject: [PATCH 30/35] Have done some testing, things seem to be working but more rigrous testing ahead --- youtube_dl/__init__.py | 405 ++++------------------------------------- 1 file changed, 40 insertions(+), 365 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 4d2c1e1b5..10bfc7790 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -2,9 +2,10 @@ # -*- coding: utf-8 -*- """ Usage: - youtube-dl [--help] [--version] [--update] [--ignore-errors] - [--dump-user-agent] [--user-agent UA] [--referer REF] - [--list-extractors] [--extractor-descriptions] [--proxy URL] + youtube-dl (--help | --version) + youtube-dl URL ... [--update] [--ignore-errors] [--dump-user-agent] + [--user-agent UA] [--referer REF] [--list-extractors] + [--extractor-descriptions] [--proxy URL] [--no-check-certificate] [--cache-dir] [--no-cache-dir] [--playlist-start NUMBER] [--playlist-end NUMBER] [--match-title REGEX] [--reject-title REGEX] @@ -12,7 +13,7 @@ Usage: [--max-filesize SIZE] [--date DATE] [--datebefore DATE] [--dateafter DATE] [--no-playlist] [--rate-limit LIMIT] [--retries RETRIES] [--buffer-size SIZE] [--no-resize-buffer] - [[--title --literal | --id] --auto-number | --output TEMPLATE] + [--title] [--id] [--auto-number] [--output TEMPLATE] [--autonumber-size NUMBER] [--restrict-filenames] [--batch-file FILE] [--no-overwrites] [--continue | --no-continue] [--cookies FILE] [--no-part] [--no-mtime] [--write-description] @@ -20,25 +21,24 @@ Usage: [--skip-download] [--get-url] [--get-title] [--get-id] [--get-thumbnail] [--get-description] [--get-filename] [--get-format] [--newline] [--no-progress] [--console-title] - [--verbose] [--dump-intermediate-pages] [--format FORMAT] - [--all-formats] [--prefer-free-formats] [--max-quality FORMAT] + [--verbose] [--dump-intermediate-pages] [--format FORMAT ] + [--all-formats] [--prefer-free-formats] [--max-quality FORMAT ] [--list-formats] [--write-sub] [--write-auto-sub] [--all-subs] - [--list-subs] [--sub-format FORMAT] [--sub-langs LANGS ...] + [--list-subs] [--sub-format FORMAT] [--sub-langs LANGS] [--username USERNAME --password PASSWORD | --netrc] [--video-password PASSWORD] [--extract-audio] [--audio-format FORMAT] [--audio-quality QUALITY] [--recode-video FORMAT] [--keep-video] [--no-post-overwrites] [--embed-subs] - URL [url ...] Options: General Options: - -h, --help print this help text and exit + -h --help print this help text and exit --version print program version and exit - -U, --update update this program to latest version. Make sure + -U --update update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed) - -i, --ignore-errors continue on download errors, for example to to + -i --ignore-errors continue on download errors, for example to to skip unavailable videos in a playlist --dump-user-agent display the current browser identification --user-agent UA specify a custom user agent @@ -49,7 +49,7 @@ Options: --extractor-descriptions Output descriptions of all supported extractors --proxy URL Use the specified HTTP/HTTPS proxy --no-check-certificate Suppress HTTPS certificate validation. - --cache-dir None Location in the filesystem where youtube-dl can + --cache-dir Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache /youtube-dl . @@ -74,8 +74,8 @@ Options: --no-playlist download only the currently playing video Download Options: - -r, --rate-limit LIMIT maximum download rate (e.g. 50k or 44.6m) - -R, --retries RETRIES number of retries [default: 10] + -r --rate-limit LIMIT maximum download rate (e.g. 50k or 44.6m) + -R --retries RETRIES number of retries [default: 10] --buffer-size SIZE size of download buffer (e.g. 1024 or 16k) [default: 1024] --no-resize-buffer do not automatically adjust the buffer size. By @@ -83,11 +83,10 @@ Options: from an initial value of SIZE. Filesystem Options: - -t, --title use title in file name (default) + -t --title use title in file name (default) --id use only video ID in file name - -l, --literal [deprecated] alias of --title - -A, --auto-number number downloaded files starting from 00000 - -o, --output TEMPLATE output filename template. Use %(title)s to get + -A --auto-number number downloaded files starting from 00000 + -o --output TEMPLATE output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, %(autonumber)s to get an automatically @@ -106,9 +105,9 @@ Options: --autonumber option is given --restrict-filenames Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames - -a, --batch-file FILE file containing URLs to download ('-' for stdin) - -w, --no-overwrites do not overwrite files - -c, --continue resume partially downloaded files + -a --batch-file FILE file containing URLs to download ('-' for stdin) + -w --no-overwrites do not overwrite files + -c --continue resume partially downloaded files --no-continue do not resume partially downloaded files (restart from beginning) --cookies FILE file to read cookies from and dump cookie jar in @@ -120,12 +119,12 @@ Options: --write-thumbnail write thumbnail image to disk Verbosity / Simulation Options: - -q, --quiet activates quiet mode - -s, --simulate do not download the video and do not write + -q --quiet activates quiet mode + -s --simulate do not download the video and do not write anything to disk --skip-download do not download the video - -g, --get-url simulate, quiet but print URL - -e, --get-title simulate, quiet but print title + -g --get-url simulate, quiet but print URL + -e --get-title simulate, quiet but print title --get-id simulate, quiet but print id --get-thumbnail simulate, quiet but print thumbnail URL --get-description simulate, quiet but print video description @@ -134,19 +133,19 @@ Options: --newline output progress bar as new lines --no-progress do not print progress bar --console-title display progress in console titlebar - -v, --verbose print various debugging information + -v --verbose print various debugging information --dump-intermediate-pages print downloaded pages to debug problems(very verbose) Video Format Options: - -f, --format FORMAT video format code, specifiy the order of + -f --format FORMAT video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported --all-formats download all available video formats --prefer-free-formats prefer free video formats unless a specific one is requested --max-quality FORMAT highest quality format to download - -F, --list-formats list all available formats (currently youtube + -F --list-formats list all available formats (currently youtube only) Subtitle Options: @@ -162,13 +161,13 @@ Options: 'en,pt' Authentication Options: - -u, --username USERNAME account username - -p, --password PASSWORD account password - -n, --netrc use .netrc authentication data + -u --username USERNAME account username + -p --password PASSWORD account password + -n --netrc use .netrc authentication data --video-password PASSWORD video password (vimeo only) Post-processing Options: - -x, --extract-audio convert video files to audio-only files (requires + -x --extract-audio convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe) --audio-format FORMAT "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; [default: best] @@ -177,7 +176,7 @@ Options: or a specific bitrate like 128K [default: 5] --recode-video FORMAT Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm) - -k, --keep-video keeps the video file on disk after the post- + -k --keep-video keeps the video file on disk after the post- processing; the video is erased by default --no-post-overwrites do not overwrite post-processed files; the post- processed files are overwritten by default @@ -244,333 +243,14 @@ from .PostProcessor import * class OptionsError(Exception): pass -def parseOpts(overrideArguments=None): - def _readOptions(filename_bytes): - try: - optionf = open(filename_bytes) - except IOError: - return [] # silently skip if file is not present - try: - res = [] - for l in optionf: - res += shlex.split(l, comments=True) - finally: - optionf.close() - return res - - def _format_option_string(option): - ''' ('-o', '--option') -> -o, --format METAVAR''' - - opts = [] - - if option._short_opts: - opts.append(option._short_opts[0]) - if option._long_opts: - opts.append(option._long_opts[0]) - if len(opts) > 1: - opts.insert(1, ', ') - - if option.takes_value(): opts.append(' %s' % option.metavar) - - return "".join(opts) - - def _comma_separated_values_options_callback(option, opt_str, value, parser): - setattr(parser.values, option.dest, value.split(',')) - - def _find_term_columns(): - columns = os.environ.get('COLUMNS', None) - if columns: - return int(columns) - - try: - sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out,err = sp.communicate() - return int(out.split()[1]) - except: - pass - return None - - def _hide_login_info(opts): - opts = list(opts) - for private_opt in ['-p', '--password', '-u', '--username']: - try: - i = opts.index(private_opt) - opts[i+1] = '' - except ValueError: - pass - return opts - - max_width = 80 - max_help_position = 80 - - # No need to wrap help messages if we're on a wide console - columns = _find_term_columns() - if columns: max_width = columns - - fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position) - fmt.format_option_strings = _format_option_string - - kw = { - 'version' : __version__, - 'formatter' : fmt, - 'usage' : '%prog [options] url [url...]', - 'conflict_handler' : 'resolve', - } - - parser = optparse.OptionParser(**kw) - - # option groups - general = optparse.OptionGroup(parser, 'General Options') - selection = optparse.OptionGroup(parser, 'Video Selection') - authentication = optparse.OptionGroup(parser, 'Authentication Options') - video_format = optparse.OptionGroup(parser, 'Video Format Options') - subtitles = optparse.OptionGroup(parser, 'Subtitle Options') - downloader = optparse.OptionGroup(parser, 'Download Options') - postproc = optparse.OptionGroup(parser, 'Post-processing Options') - filesystem = optparse.OptionGroup(parser, 'Filesystem Options') - verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') - - #general.add_option('-h', '--help', - # action='help', help='print this help text and exit') - #general.add_option('-v', '--version', - # action='version', help='print program version and exit') - #general.add_option('-U', '--update', - # action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)') - #general.add_option('-i', '--ignore-errors', - # action='store_true', dest='ignoreerrors', help='continue on download errors', default=False) - #general.add_option('--dump-user-agent', - # action='store_true', dest='dump_user_agent', - # help='display the current browser identification', default=False) - #general.add_option('--user-agent', - # dest='user_agent', help='specify a custom user agent', metavar='UA') - #general.add_option('--referer', - # dest='referer', help='specify a custom referer, use if the video access is restricted to one domain', - # metavar='REF', default=None) - #general.add_option('--list-extractors', - # action='store_true', dest='list_extractors', - # help='List all supported extractors and the URLs they would handle', default=False) - #general.add_option('--extractor-descriptions', - # action='store_true', dest='list_extractor_descriptions', - # help='Output descriptions of all supported extractors', default=False) - #general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL') - #general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.') - - - #selection.add_option('--playlist-start', - # dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1) - #selection.add_option('--playlist-end', - # dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) - #selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)') - #selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)') - #selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None) - #selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None) - #selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None) - #selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None) - #selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None) - #selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None) - - - #authentication.add_option('-u', '--username', - # dest='username', metavar='USERNAME', help='account username') - #authentication.add_option('-p', '--password', - # dest='password', metavar='PASSWORD', help='account password') - #authentication.add_option('-n', '--netrc', - # action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) - #authentication.add_option('--video-password', - # dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)') - - - #video_format.add_option('-f', '--format', - # action='store', dest='format', metavar='FORMAT', - # help='video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported') - #video_format.add_option('--all-formats', - # action='store_const', dest='format', help='download all available video formats', const='all') - #video_format.add_option('--prefer-free-formats', - # action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested') - #video_format.add_option('--max-quality', - # action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') - #video_format.add_option('-F', '--list-formats', - # action='store_true', dest='listformats', help='list all available formats (currently youtube only)') - - #subtitles.add_option('--write-sub', '--write-srt', - # action='store_true', dest='writesubtitles', - # help='write subtitle file', default=False) - #subtitles.add_option('--write-auto-sub', '--write-automatic-sub', - # action='store_true', dest='writeautomaticsub', - # help='write automatic subtitle file (youtube only)', default=False) - #subtitles.add_option('--all-subs', - # action='store_true', dest='allsubtitles', - # help='downloads all the available subtitles of the video', default=False) - #subtitles.add_option('--list-subs', - # action='store_true', dest='listsubtitles', - # help='lists all available subtitles for the video', default=False) - #subtitles.add_option('--sub-format', - # action='store', dest='subtitlesformat', metavar='FORMAT', - # help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt') - #subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang', - # action='callback', dest='subtitleslangs', metavar='LANGS', type='str', - # default=[], callback=_comma_separated_values_options_callback, - # help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'') - - #downloader.add_option('-r', '--rate-limit', - # dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)') - #downloader.add_option('-R', '--retries', - # dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10) - #downloader.add_option('--buffer-size', - # dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024") - #downloader.add_option('--no-resize-buffer', - # action='store_true', dest='noresizebuffer', - # help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False) - #downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP) - - #verbosity.add_option('-q', '--quiet', - # action='store_true', dest='quiet', help='activates quiet mode', default=False) - #verbosity.add_option('-s', '--simulate', - # action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False) - #verbosity.add_option('--skip-download', - # action='store_true', dest='skip_download', help='do not download the video', default=False) - #verbosity.add_option('-g', '--get-url', - # action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False) - #verbosity.add_option('-e', '--get-title', - # action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False) - #verbosity.add_option('--get-id', - # action='store_true', dest='getid', help='simulate, quiet but print id', default=False) - #verbosity.add_option('--get-thumbnail', - # action='store_true', dest='getthumbnail', - # help='simulate, quiet but print thumbnail URL', default=False) - #verbosity.add_option('--get-description', - # action='store_true', dest='getdescription', - # help='simulate, quiet but print video description', default=False) - #verbosity.add_option('--get-filename', - # action='store_true', dest='getfilename', - # help='simulate, quiet but print output filename', default=False) - #verbosity.add_option('--get-format', - # action='store_true', dest='getformat', - # help='simulate, quiet but print output format', default=False) - #verbosity.add_option('--newline', - # action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False) - #verbosity.add_option('--no-progress', - # action='store_true', dest='noprogress', help='do not print progress bar', default=False) - #verbosity.add_option('--console-title', - # action='store_true', dest='consoletitle', - # help='display progress in console titlebar', default=False) - #verbosity.add_option('-v', '--verbose', - # action='store_true', dest='verbose', help='print various debugging information', default=False) - #verbosity.add_option('--dump-intermediate-pages', - # action='store_true', dest='dump_intermediate_pages', default=False, - # help='print downloaded pages to debug problems(very verbose)') - - #filesystem.add_option('-t', '--title', - # action='store_true', dest='usetitle', help='use title in file name (default)', default=False) - #filesystem.add_option('--id', - # action='store_true', dest='useid', help='use only video ID in file name', default=False) - #filesystem.add_option('-l', '--literal', - # action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False) - #filesystem.add_option('-A', '--auto-number', - # action='store_true', dest='autonumber', - # help='number downloaded files starting from 00000', default=False) - #filesystem.add_option('-o', '--output', - # dest='outtmpl', metavar='TEMPLATE', - # help=('output filename template. Use %(title)s to get the title, ' - # '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, ' - # '%(autonumber)s to get an automatically incremented number, ' - # '%(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), ' - # '%(extractor)s for the provider (youtube, metacafe, etc), ' - # '%(id)s for the video id , %(playlist)s for the playlist the video is in, ' - # '%(playlist_index)s for the position in the playlist and %% for a literal percent. ' - # 'Use - to output to stdout. Can also be used to download to a different directory, ' - # 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')) - #filesystem.add_option('--autonumber-size', - # dest='autonumber_size', metavar='NUMBER', - # help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given') - #filesystem.add_option('--restrict-filenames', - # action='store_true', dest='restrictfilenames', - # help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False) - #filesystem.add_option('-a', '--batch-file', - # dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') - #filesystem.add_option('-w', '--no-overwrites', - # action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) - #filesystem.add_option('-c', '--continue', - # action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True) - #filesystem.add_option('--no-continue', - # action='store_false', dest='continue_dl', - # help='do not resume partially downloaded files (restart from beginning)') - #filesystem.add_option('--cookies', - # dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in') - #filesystem.add_option('--no-part', - # action='store_true', dest='nopart', help='do not use .part files', default=False) - #filesystem.add_option('--no-mtime', - # action='store_false', dest='updatetime', - # help='do not use the Last-modified header to set the file modification time', default=True) - #filesystem.add_option('--write-description', - # action='store_true', dest='writedescription', - # help='write video description to a .description file', default=False) - #filesystem.add_option('--write-info-json', - # action='store_true', dest='writeinfojson', - # help='write video metadata to a .info.json file', default=False) - #filesystem.add_option('--write-thumbnail', - # action='store_true', dest='writethumbnail', - # help='write thumbnail image to disk', default=False) - - - #postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False, - # help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)') - #postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', - # help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default') - #postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5', - # help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)') - #postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None, - # help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)') - #postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, - # help='keeps the video file on disk after the post-processing; the video is erased by default') - #postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False, - # help='do not overwrite post-processed files; the post-processed files are overwritten by default') - postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False, - help='embed subtitles in the video (only for mp4 videos)') - - - parser.add_option_group(general) - parser.add_option_group(selection) - parser.add_option_group(downloader) - parser.add_option_group(filesystem) - parser.add_option_group(verbosity) - parser.add_option_group(video_format) - parser.add_option_group(subtitles) - parser.add_option_group(authentication) - parser.add_option_group(postproc) - - if overrideArguments is not None: - opts, args = parser.parse_args(overrideArguments) - if opts['--verbose']: - sys.stderr.write(u'[debug] Override config: ' + repr(overrideArguments) + '\n') - else: - xdg_config_home = os.environ.get('XDG_CONFIG_HOME') - if xdg_config_home: - userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf') - else: - userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf') - systemConf = _readOptions('/etc/youtube-dl.conf') - userConf = _readOptions(userConfFile) - commandLineConf = sys.argv[1:] - argv = systemConf + userConf + commandLineConf - opts, args = parser.parse_args(argv) - if opts['--verbose']: - sys.stderr.write(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n') - sys.stderr.write(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n') - sys.stderr.write(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n') - - return parser, opts, args - def _real_main(argv=None): # Compatibility fixes for Windows if sys.platform == 'win32': # https://github.com/rg3/youtube-dl/issues/820 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) - opts = docopt(__doc__, version='0.0.1') - #Support both --title and deprecated --literal - if not opts['--title']: - opts['--title'] = opts['--literal'] + opts = docopt(__doc__, version=__version__) + raw_input(opts) #--all-formats will supersede --format if opts['--all-formats']: opts['--format'] = 'all' @@ -583,8 +263,6 @@ def _real_main(argv=None): #Invert boolean value of opts['--no-mtime'] opts['--no-mtime'] = not opts['--no-mtime'] - #parser, opts, args = parseOpts(argv) - # Open appropriate CookieJar if not opts['--cookies']: jar = compat_cookiejar.CookieJar() @@ -626,7 +304,8 @@ def _real_main(argv=None): sys.stderr.write(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n') except IOError: sys.exit(u'ERROR: batch file could not be read') - all_urls = batchurls + args + all_urls = batchurls + opts['URL'] + print(all_urls) all_urls = [url.strip() for url in all_urls] # General configuration @@ -675,18 +354,14 @@ def _real_main(argv=None): # Conflicting, missing and erroneous options - #if opts['--netrc'] and (opts['--username'] or opts['--password']): - # parser.error(u'using .netrc conflicts with giving username/password') if opts['--password'] and not opts['--username']: raise OptionsError(u' account username missing\n') if opts['--username'] and not opts['--password']: opts['--password'] = getpass.getpass(u'Type account password and press return:') - - #if opts['--output'] is not None and (opts['--title'] or opts['--auto-number'] or opts['--id']): - # raise OptionsError(u'using output template conflicts with using title, video ID or auto number') - #if optsopts['--title'] and opts.useid: - # parser.error(u'using title conflicts with using video ID') - + if opts['--output'] and (opts['--title'] or opts['--auto-number'] or opts['--id']): + raise OptionsError(u'using output template conflicts with using title, video ID or auto number') + if opts['--title'] and opts['--id']: + raise OptionsError(u'using title conflicts with using video ID') if opts['--rate-limit']: numeric_limit = FileDownloader.parse_bytes(opts['--rate-limit']) if numeric_limit is None: From 0e5332f3462c24bc840fbdb53fe8841d8721fab1 Mon Sep 17 00:00:00 2001 From: SavinaRoja Date: Sun, 6 Oct 2013 19:32:02 -0400 Subject: [PATCH 31/35] Trying to work out default values --- youtube_dl/__init__.py | 111 ++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 10bfc7790..9dda204d6 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -4,31 +4,31 @@ Usage: youtube-dl (--help | --version) youtube-dl URL ... [--update] [--ignore-errors] [--dump-user-agent] - [--user-agent UA] [--referer REF] [--list-extractors] - [--extractor-descriptions] [--proxy URL] + [--user-agent=UA] [--referer REF] [--list-extractors] + [--extractor-descriptions] [--proxy=URL] [--no-check-certificate] [--cache-dir] [--no-cache-dir] - [--playlist-start NUMBER] [--playlist-end NUMBER] - [--match-title REGEX] [--reject-title REGEX] - [--max-downloads NUMBER] [--min-filesize SIZE] - [--max-filesize SIZE] [--date DATE] [--datebefore DATE] - [--dateafter DATE] [--no-playlist] [--rate-limit LIMIT] - [--retries RETRIES] [--buffer-size SIZE] [--no-resize-buffer] - [--title] [--id] [--auto-number] [--output TEMPLATE] - [--autonumber-size NUMBER] [--restrict-filenames] - [--batch-file FILE] [--no-overwrites] [--continue | --no-continue] - [--cookies FILE] [--no-part] [--no-mtime] [--write-description] + [--playlist-start=NUMBER] [--playlist-end=NUMBER] + [--match-title=REGEX] [--reject-title=REGEX] + [--max-downloads=NUMBER] [--min-filesize=SIZE] + [--max-filesize=SIZE] [--date=DATE] [--datebefore=DATE] + [--dateafter=DATE] [--no-playlist] [--rate-limit=LIMIT] + [--retries=RETRIES] [--buffer-size=SIZE] [--no-resize-buffer] + [--title] [--id] [--auto-number] [--output=TEMPLATE] + [--autonumber-size=NUMBER] [--restrict-filenames] + [--batch-file=FILE] [--no-overwrites] [--continue | --no-continue] + [--cookies=FILE] [--no-part] [--no-mtime] [--write-description] [--write-info-json] [--write-thumbnail] [--quiet] [--simulate] [--skip-download] [--get-url] [--get-title] [--get-id] [--get-thumbnail] [--get-description] [--get-filename] [--get-format] [--newline] [--no-progress] [--console-title] - [--verbose] [--dump-intermediate-pages] [--format FORMAT ] - [--all-formats] [--prefer-free-formats] [--max-quality FORMAT ] + [--verbose] [--dump-intermediate-pages] [--format=FORMAT ] + [--all-formats] [--prefer-free-formats] [--max-quality=FORMAT ] [--list-formats] [--write-sub] [--write-auto-sub] [--all-subs] - [--list-subs] [--sub-format FORMAT] [--sub-langs LANGS] - [--username USERNAME --password PASSWORD | --netrc] - [--video-password PASSWORD] [--extract-audio] - [--audio-format FORMAT] [--audio-quality QUALITY] - [--recode-video FORMAT] [--keep-video] [--no-post-overwrites] + [--list-subs] [--sub-format=FORMAT] [--sub-langs=LANGS...] + [--username=USERNAME --password=PASSWORD | --netrc] + [--video-password=PASSWORD] [--extract-audio] + [--audio-format=FORMAT] [--audio-quality=QUALITY] + [--recode-video=FORMAT] [--keep-video] [--no-post-overwrites] [--embed-subs] Options: @@ -41,13 +41,13 @@ Options: -i --ignore-errors continue on download errors, for example to to skip unavailable videos in a playlist --dump-user-agent display the current browser identification - --user-agent UA specify a custom user agent - --referer REF specify a custom referer, use if the video access + --user-agent=UA specify a custom user agent + --referer=REF specify a custom referer, use if the video access is restricted to one domain --list-extractors List all supported extractors and the URLs they would handle --extractor-descriptions Output descriptions of all supported extractors - --proxy URL Use the specified HTTP/HTTPS proxy + --proxy=URL Use the specified HTTP/HTTPS proxy --no-check-certificate Suppress HTTPS certificate validation. --cache-dir Location in the filesystem where youtube-dl can store downloaded information permanently. By @@ -56,37 +56,37 @@ Options: --no-cache-dir Disable filesystem caching Video Selection: - --playlist-start NUMBER playlist video to start at [default: 1] - --playlist-end NUMBER playlist video to end at (defaults to last) + --playlist-start=NUMBER playlist video to start at [default: 1] + --playlist-end=NUMBER playlist video to end at (defaults to last) [default: -1] - --match-title REGEX download only matching titles (regex or caseless + --match-title=REGEX download only matching titles (regex or caseless sub-string) - --reject-title REGEX skip download for matching titles (regex or + --reject-title=REGEX skip download for matching titles (regex or caseless sub-string) - --max-downloads NUMBER Abort after downloading NUMBER files - --min-filesize SIZE Do not download any videos smaller than SIZE + --max-downloads=NUMBER Abort after downloading NUMBER files + --min-filesize=SIZE Do not download any videos smaller than SIZE (e.g. 50k or 44.6m) - --max-filesize SIZE Do not download any videos larger than SIZE (e.g. + --max-filesize=SIZE Do not download any videos larger than SIZE (e.g. 50k or 44.6m) - --date DATE download only videos uploaded in this date - --datebefore DATE download only videos uploaded before this date - --dateafter DATE download only videos uploaded after this date + --date=DATE download only videos uploaded in this date + --datebefore=DATE download only videos uploaded before this date + --dateafter=DATE download only videos uploaded after this date --no-playlist download only the currently playing video Download Options: - -r --rate-limit LIMIT maximum download rate (e.g. 50k or 44.6m) - -R --retries RETRIES number of retries [default: 10] - --buffer-size SIZE size of download buffer (e.g. 1024 or 16k) + -r --rate-limit=LIMIT maximum download rate (e.g. 50k or 44.6m) + -R --retries=RETRIES number of retries [default: 10] + --buffer-size=SIZE size of download buffer (e.g. 1024 or 16k) [default: 1024] --no-resize-buffer do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE. Filesystem Options: - -t --title use title in file name (default) + -t --title use title in file name (default action) --id use only video ID in file name -A --auto-number number downloaded files starting from 00000 - -o --output TEMPLATE output filename template. Use %(title)s to get + -o --output=TEMPLATE output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, %(autonumber)s to get an automatically @@ -100,17 +100,17 @@ Options: output to stdout. Can also be used to download to a different directory, for example with -o '/my/d ownloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' . - --autonumber-size NUMBER Specifies the number of digits in %(autonumber)s + --autonumber-size=NUMBER Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given --restrict-filenames Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames - -a --batch-file FILE file containing URLs to download ('-' for stdin) + -a --batch-file=FILE file containing URLs to download ('-' for stdin) -w --no-overwrites do not overwrite files -c --continue resume partially downloaded files --no-continue do not resume partially downloaded files (restart from beginning) - --cookies FILE file to read cookies from and dump cookie jar in + --cookies=FILE file to read cookies from and dump cookie jar in --no-part do not use .part files --no-mtime do not use the Last-modified header to set the file modification time @@ -138,13 +138,13 @@ Options: verbose) Video Format Options: - -f --format FORMAT video format code, specifiy the order of + -f --format=FORMAT video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported --all-formats download all available video formats --prefer-free-formats prefer free video formats unless a specific one is requested - --max-quality FORMAT highest quality format to download + --max-quality=FORMAT highest quality format to download -F --list-formats list all available formats (currently youtube only) @@ -154,27 +154,27 @@ Options: --all-subs downloads all the available subtitles of the video --list-subs lists all available subtitles for the video - --sub-format FORMAT subtitle format ([sbv/vtt] youtube only) + --sub-format=FORMAT subtitle format ([sbv/vtt] youtube only) [default: srt] - --sub-langs LANGS ... languages of the subtitles to download (optional, + --sub-langs=LANGS ... languages of the subtitles to download (optional, multiple arguments) use IETF language tags like 'en,pt' Authentication Options: - -u --username USERNAME account username - -p --password PASSWORD account password + -u --username=USERNAME account username + -p --password=PASSWORD account password -n --netrc use .netrc authentication data - --video-password PASSWORD video password (vimeo only) + --video-password=PASSWORD video password (vimeo only) Post-processing Options: -x --extract-audio convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe) - --audio-format FORMAT "best", "aac", "vorbis", "mp3", "m4a", "opus", or + --audio-format=FORMAT "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; [default: best] - --audio-quality QUALITY ffmpeg/avconv audio quality specification, insert + --audio-quality=QUALITY ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K [default: 5] - --recode-video FORMAT Encode the video to another format if necessary + --recode-video=FORMAT Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm) -k --keep-video keeps the video file on disk after the post- processing; the video is erased by default @@ -220,15 +220,14 @@ __license__ = 'Public Domain' import codecs import getpass -#import optparse import os import random import re -import shlex +#import shlex import socket import subprocess import sys -import warnings +#import warnings import platform from .docopt import docopt @@ -243,14 +242,14 @@ from .PostProcessor import * class OptionsError(Exception): pass -def _real_main(argv=None): +def _real_main(): # Compatibility fixes for Windows if sys.platform == 'win32': # https://github.com/rg3/youtube-dl/issues/820 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) opts = docopt(__doc__, version=__version__) - raw_input(opts) + print(opts) #--all-formats will supersede --format if opts['--all-formats']: opts['--format'] = 'all' @@ -543,7 +542,7 @@ def _real_main(argv=None): def main(argv=None): try: - _real_main(argv) + _real_main() except DownloadError: sys.exit(1) except SameFileError: From 827572dfa8854f0356c4b77671687a78a53e7777 Mon Sep 17 00:00:00 2001 From: Paul Barton Date: Mon, 7 Oct 2013 15:37:58 -0400 Subject: [PATCH 32/35] Removing TODO message --- youtube_dl/YoutubeDL.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index e78cb394a..30643010f 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -16,8 +16,6 @@ from .utils import * from .extractor import get_info_extractor, gen_extractors from .FileDownloader import FileDownloader -#TODO: Check for parameters that eval None identity, docopt defaults to False - class YoutubeDL(object): """YoutubeDL class. From c634a6ac421beaf5dc6b564097415072882f9a17 Mon Sep 17 00:00:00 2001 From: Paul Barton Date: Mon, 7 Oct 2013 15:38:40 -0400 Subject: [PATCH 33/35] docopt parsing now works and I have tested a small subset of command line options for proper working --- youtube_dl/__init__.py | 35 ++++++----------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 9dda204d6..ded6ea1ef 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -3,33 +3,7 @@ """ Usage: youtube-dl (--help | --version) - youtube-dl URL ... [--update] [--ignore-errors] [--dump-user-agent] - [--user-agent=UA] [--referer REF] [--list-extractors] - [--extractor-descriptions] [--proxy=URL] - [--no-check-certificate] [--cache-dir] [--no-cache-dir] - [--playlist-start=NUMBER] [--playlist-end=NUMBER] - [--match-title=REGEX] [--reject-title=REGEX] - [--max-downloads=NUMBER] [--min-filesize=SIZE] - [--max-filesize=SIZE] [--date=DATE] [--datebefore=DATE] - [--dateafter=DATE] [--no-playlist] [--rate-limit=LIMIT] - [--retries=RETRIES] [--buffer-size=SIZE] [--no-resize-buffer] - [--title] [--id] [--auto-number] [--output=TEMPLATE] - [--autonumber-size=NUMBER] [--restrict-filenames] - [--batch-file=FILE] [--no-overwrites] [--continue | --no-continue] - [--cookies=FILE] [--no-part] [--no-mtime] [--write-description] - [--write-info-json] [--write-thumbnail] [--quiet] [--simulate] - [--skip-download] [--get-url] [--get-title] [--get-id] - [--get-thumbnail] [--get-description] [--get-filename] - [--get-format] [--newline] [--no-progress] [--console-title] - [--verbose] [--dump-intermediate-pages] [--format=FORMAT ] - [--all-formats] [--prefer-free-formats] [--max-quality=FORMAT ] - [--list-formats] [--write-sub] [--write-auto-sub] [--all-subs] - [--list-subs] [--sub-format=FORMAT] [--sub-langs=LANGS...] - [--username=USERNAME --password=PASSWORD | --netrc] - [--video-password=PASSWORD] [--extract-audio] - [--audio-format=FORMAT] [--audio-quality=QUALITY] - [--recode-video=FORMAT] [--keep-video] [--no-post-overwrites] - [--embed-subs] + youtube-dl URL ... [options] Options: General Options: @@ -55,7 +29,7 @@ Options: /youtube-dl . --no-cache-dir Disable filesystem caching - Video Selection: + Video Selection Options: --playlist-start=NUMBER playlist video to start at [default: 1] --playlist-end=NUMBER playlist video to end at (defaults to last) [default: -1] @@ -123,7 +97,7 @@ Options: -s --simulate do not download the video and do not write anything to disk --skip-download do not download the video - -g --get-url simulate, quiet but print URL + -g --get-url simulate, quiet but print URL -e --get-title simulate, quiet but print title --get-id simulate, quiet but print id --get-thumbnail simulate, quiet but print thumbnail URL @@ -136,6 +110,7 @@ Options: -v --verbose print various debugging information --dump-intermediate-pages print downloaded pages to debug problems(very verbose) + --test Download only first bytes to test the downloader Video Format Options: -f --format=FORMAT video format code, specifiy the order of @@ -353,6 +328,8 @@ def _real_main(): # Conflicting, missing and erroneous options + if opts['--netrc'] and (opts['--username'] or opts['--password']): + raise OptionsError(u'using .netrc conflicts with giving username/password') if opts['--password'] and not opts['--username']: raise OptionsError(u' account username missing\n') if opts['--username'] and not opts['--password']: From 6c8a43c45a67506bf9a505c498478ca3afabb381 Mon Sep 17 00:00:00 2001 From: Paul Barton Date: Mon, 7 Oct 2013 15:47:38 -0400 Subject: [PATCH 34/35] fixed alignment in docstring for --get-url --- youtube_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index ded6ea1ef..82a647160 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -97,7 +97,7 @@ Options: -s --simulate do not download the video and do not write anything to disk --skip-download do not download the video - -g --get-url simulate, quiet but print URL + -g --get-url simulate, quiet but print URL -e --get-title simulate, quiet but print title --get-id simulate, quiet but print id --get-thumbnail simulate, quiet but print thumbnail URL From b099a18e69849ff1b005994382827f338efe431b Mon Sep 17 00:00:00 2001 From: Paul Barton Date: Mon, 7 Oct 2013 16:01:32 -0400 Subject: [PATCH 35/35] removing printing of opts --- youtube_dl/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 82a647160..84026e759 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -224,7 +224,6 @@ def _real_main(): codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) opts = docopt(__doc__, version=__version__) - print(opts) #--all-formats will supersede --format if opts['--all-formats']: opts['--format'] = 'all'