diff --git a/youtube-dl b/youtube-dl index dcbecd336..19c327e97 100755 --- a/youtube-dl +++ b/youtube-dl @@ -4,9 +4,12 @@ # Author: Danny Colligan # Author: Benjamin Johnson # Author: Vasyl' Vavrychuk +# Author: Witold Baryluk # License: Public domain code import cookielib +import ctypes import datetime +import gzip import htmlentitydefs import httplib import locale @@ -17,11 +20,13 @@ import os.path import re import socket import string +import StringIO import subprocess import sys import time import urllib import urllib2 +import zlib # parse_qs was moved from the cgi module to the urlparse module recently. try: @@ -57,7 +62,7 @@ def preferredencoding(): def htmlentity_transform(matchobj): """Transforms an HTML entity to a Unicode character. - + This function receives a match object and is intended to be used with the re.sub() function. """ @@ -114,7 +119,7 @@ def sanitize_open(filename, open_mode): class DownloadError(Exception): """Download Error exception. - + This exception may be thrown by FileDownloader objects if they are not configured to continue on errors. They will contain the appropriate error message. @@ -160,6 +165,64 @@ class ContentTooShortError(Exception): self.downloaded = downloaded self.expected = expected +class YoutubeDLHandler(urllib2.HTTPHandler): + """Handler for HTTP requests and responses. + + This class, when installed with an OpenerDirector, automatically adds + the standard headers to every HTTP request and handles gzipped and + deflated responses from web servers. If compression is to be avoided in + a particular request, the original request in the program code only has + to include the HTTP header "Youtubedl-No-Compression", which will be + removed before making the real request. + + Part of this code was copied from: + + http://techknack.net/python-urllib2-handlers/ + + Andrew Rowls, the author of that code, agreed to release it to the + public domain. + """ + + @staticmethod + def deflate(data): + try: + return zlib.decompress(data, -zlib.MAX_WBITS) + except zlib.error: + return zlib.decompress(data) + + @staticmethod + def addinfourl_wrapper(stream, headers, url, code): + if hasattr(urllib2.addinfourl, 'getcode'): + return urllib2.addinfourl(stream, headers, url, code) + ret = urllib2.addinfourl(stream, headers, url) + ret.code = code + return ret + + def http_request(self, req): + for h in std_headers: + if h in req.headers: + del req.headers[h] + req.add_header(h, std_headers[h]) + if 'Youtubedl-no-compression' in req.headers: + if 'Accept-encoding' in req.headers: + del req.headers['Accept-encoding'] + del req.headers['Youtubedl-no-compression'] + return req + + def http_response(self, req, resp): + old_resp = resp + # gzip + if resp.headers.get('Content-encoding', '') == 'gzip': + gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r') + resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) + resp.msg = old_resp.msg + # deflate + if resp.headers.get('Content-encoding', '') == 'deflate': + gz = StringIO.StringIO(self.deflate(resp.read())) + resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) + resp.msg = old_resp.msg + return resp + class FileDownloader(object): """File Downloader class. @@ -208,6 +271,8 @@ class FileDownloader(object): playliststart: Playlist item to start at. playlistend: Playlist item to end at. logtostderr: Log messages to stderr instead of stdout. + consoletitle: Display progress in console window's titlebar. + nopart: Do not use temporary .part files. """ params = None @@ -225,7 +290,7 @@ class FileDownloader(object): self._num_downloads = 0 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self.params = params - + @staticmethod def pmkdir(filename): """Create directory components in filename. Similar to Unix "mkdir -p".""" @@ -235,14 +300,7 @@ class FileDownloader(object): for dir in aggregate: if not os.path.exists(dir): os.mkdir(dir) - - @staticmethod - def temp_name(filename): - """Returns a temporary filename for the given filename.""" - if filename == u'-' or (os.path.exists(filename) and not os.path.isfile(filename)): - return filename - return filename + u'.part' - + @staticmethod def format_bytes(bytes): if bytes is None: @@ -311,12 +369,12 @@ class FileDownloader(object): """Add an InfoExtractor object to the end of the list.""" self._ies.append(ie) ie.set_downloader(self) - + def add_post_processor(self, pp): """Add a PostProcessor object to the end of the chain.""" self._pps.append(pp) pp.set_downloader(self) - + def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False): """Print message to stdout if not in quiet mode.""" try: @@ -327,11 +385,22 @@ class FileDownloader(object): except (UnicodeEncodeError), err: if not ignore_encoding_errors: raise - + def to_stderr(self, message): """Print message to stderr.""" print >>sys.stderr, message.encode(preferredencoding()) - + + def to_cons_title(self, message): + """Set console/terminal window title to message.""" + if not self.params.get('consoletitle', False): + return + if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow(): + # c_wchar_p() might not be necessary if `message` is + # already of type unicode() + ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) + elif 'TERM' in os.environ: + sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding())) + def fixed_template(self): """Checks if the output template is fixed.""" return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None) @@ -361,7 +430,19 @@ class FileDownloader(object): speed = float(byte_counter) / elapsed if speed > rate_limit: time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) - + + def temp_name(self, filename): + """Returns a temporary filename for the given filename.""" + if self.params.get('nopart', False) or filename == u'-' or \ + (os.path.exists(filename) and not os.path.isfile(filename)): + return filename + return filename + u'.part' + + def undo_temp_name(self, filename): + if filename.endswith(u'.part'): + return filename[:-len(u'.part')] + return filename + def try_rename(self, old_filename, new_filename): try: if old_filename == new_filename: @@ -373,40 +454,42 @@ class FileDownloader(object): def report_destination(self, filename): """Report destination filename.""" self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True) - + def report_progress(self, percent_str, data_len_str, speed_str, eta_str): """Report download progress.""" if self.params.get('noprogress', False): return self.to_screen(u'\r[download] %s of %s at %s ETA %s' % (percent_str, data_len_str, speed_str, eta_str), skip_eol=True) + self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' % + (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip())) def report_resuming_byte(self, resume_len): """Report attempt to resume at given byte.""" self.to_screen(u'[download] Resuming download at byte %s' % resume_len) - + def report_retry(self, count, retries): """Report retry in case of HTTP error 5xx""" self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) - + def report_file_already_downloaded(self, file_name): """Report file has already been fully downloaded.""" try: self.to_screen(u'[download] %s has already been downloaded' % file_name) except (UnicodeEncodeError), err: self.to_screen(u'[download] The file has already been downloaded') - + def report_unable_to_resume(self): """Report it was impossible to resume download.""" self.to_screen(u'[download] Unable to resume') - + def report_finish(self): """Report download finished.""" if self.params.get('noprogress', False): self.to_screen(u'[download] Download completed') else: self.to_screen(u'') - + def increment_downloads(self): """Increment the ordinal that assigns a number to each file.""" self._num_downloads += 1 @@ -426,7 +509,7 @@ class FileDownloader(object): print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace') return - + try: template_dict = dict(info_dict) template_dict['epoch'] = unicode(long(time.time())) @@ -497,7 +580,7 @@ class FileDownloader(object): info = pp.run(info) if info is None: break - + def _download_with_rtmpdump(self, filename, url, player_url): self.report_destination(filename) tmpfilename = self.temp_name(filename) @@ -532,7 +615,7 @@ class FileDownloader(object): def _do_download(self, filename, url, player_url): # Check file already present - if self.params.get('continuedl', False) and os.path.isfile(filename): + if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False): self.report_file_already_downloaded(filename) return True @@ -543,8 +626,11 @@ class FileDownloader(object): tmpfilename = self.temp_name(filename) stream = None open_mode = 'wb' - basic_request = urllib2.Request(url, None, std_headers) - request = urllib2.Request(url, None, std_headers) + + # Do not include the Accept-Encoding header + headers = {'Youtubedl-no-compression': 'True'} + basic_request = urllib2.Request(url, None, headers) + request = urllib2.Request(url, None, headers) # Establish possible resume length if os.path.isfile(tmpfilename): @@ -626,6 +712,7 @@ class FileDownloader(object): if stream is None: try: (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) + filename = self.undo_temp_name(tmpfilename) self.report_destination(filename) except (OSError, IOError), err: self.trouble(u'ERROR: unable to open for writing: %s' % str(err)) @@ -715,7 +802,7 @@ class InfoExtractor(object): def set_downloader(self, downloader): """Sets the downloader for this IE.""" self._downloader = downloader - + def _real_initialize(self): """Real initialization process. Redefine in subclasses.""" pass @@ -727,7 +814,7 @@ class InfoExtractor(object): class YoutubeIE(InfoExtractor): """Information extractor for youtube.com.""" - _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/(?:(?:v/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))))?([0-9A-Za-z_-]+)(?(1).+)?$' + _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$' _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en' _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' @@ -756,31 +843,31 @@ class YoutubeIE(InfoExtractor): def report_login(self): """Report attempt to log in.""" self._downloader.to_screen(u'[youtube] Logging in') - + def report_age_confirmation(self): """Report attempt to confirm age.""" self._downloader.to_screen(u'[youtube] Confirming age') - + def report_video_webpage_download(self, video_id): """Report attempt to download video webpage.""" self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id) - + def report_video_info_webpage_download(self, video_id): """Report attempt to download video info webpage.""" self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id) - + def report_information_extraction(self, video_id): """Report attempt to extract video information.""" self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id) - + def report_unavailable_format(self, video_id, format): """Report extracted video URL.""" self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format)) - + def report_rtmp_download(self): """Indicate the download will use the RTMP protocol.""" self._downloader.to_screen(u'[youtube] RTMP download detected') - + def _real_initialize(self): if self._downloader is None: return @@ -806,7 +893,7 @@ class YoutubeIE(InfoExtractor): return # Set language - request = urllib2.Request(self._LANG_URL, None, std_headers) + request = urllib2.Request(self._LANG_URL) try: self.report_lang() urllib2.urlopen(request).read() @@ -826,7 +913,7 @@ class YoutubeIE(InfoExtractor): 'username': username, 'password': password, } - request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers) + request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form)) try: self.report_login() login_results = urllib2.urlopen(request).read() @@ -836,13 +923,13 @@ class YoutubeIE(InfoExtractor): except (urllib2.URLError, httplib.HTTPException, socket.error), err: self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err)) return - + # Confirm age age_form = { 'next_url': '/', 'action_confirm': 'Confirm', } - request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers) + request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form)) try: self.report_age_confirmation() age_results = urllib2.urlopen(request).read() @@ -860,7 +947,7 @@ class YoutubeIE(InfoExtractor): # Get video webpage self.report_video_webpage_download(video_id) - request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id, None, std_headers) + request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id) try: video_webpage = urllib2.urlopen(request).read() except (urllib2.URLError, httplib.HTTPException, socket.error), err: @@ -879,7 +966,7 @@ class YoutubeIE(InfoExtractor): for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']: video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en' % (video_id, el_type)) - request = urllib2.Request(video_info_url, None, std_headers) + request = urllib2.Request(video_info_url) try: video_info_webpage = urllib2.urlopen(request).read() video_info = parse_qs(video_info_webpage) @@ -1002,7 +1089,7 @@ class YoutubeIE(InfoExtractor): 'player_url': player_url, }) except UnavailableVideoError, err: - self._downloader.trouble(u'ERROR: unable to download video') + self._downloader.trouble(u'\nERROR: unable to download video') class MetacafeIE(InfoExtractor): @@ -1028,18 +1115,18 @@ class MetacafeIE(InfoExtractor): def report_age_confirmation(self): """Report attempt to confirm age.""" self._downloader.to_screen(u'[metacafe] Confirming age') - + def report_download_webpage(self, video_id): """Report webpage download.""" self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id) - + def report_extraction(self, video_id): """Report information extraction.""" self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id) def _real_initialize(self): # Retrieve disclaimer - request = urllib2.Request(self._DISCLAIMER, None, std_headers) + request = urllib2.Request(self._DISCLAIMER) try: self.report_disclaimer() disclaimer = urllib2.urlopen(request).read() @@ -1052,14 +1139,14 @@ class MetacafeIE(InfoExtractor): 'filters': '0', 'submit': "Continue - I'm over 18", } - request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers) + request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form)) try: self.report_age_confirmation() disclaimer = urllib2.urlopen(request).read() except (urllib2.URLError, httplib.HTTPException, socket.error), err: self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err)) return - + def _real_extract(self, url): # Extract id and simplified title from URL mobj = re.match(self._VALID_URL, url) @@ -1095,7 +1182,7 @@ class MetacafeIE(InfoExtractor): if mobj is not None: mediaURL = urllib.unquote(mobj.group(1)) video_extension = mediaURL[-3:] - + # Extract gdaKey if available mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage) if mobj is None: @@ -1147,7 +1234,7 @@ class MetacafeIE(InfoExtractor): 'player_url': None, }) except UnavailableVideoError: - self._downloader.trouble(u'ERROR: unable to download video') + self._downloader.trouble(u'\nERROR: unable to download video') class DailymotionIE(InfoExtractor): @@ -1165,7 +1252,7 @@ class DailymotionIE(InfoExtractor): def report_download_webpage(self, video_id): """Report webpage download.""" self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id) - + def report_extraction(self, video_id): """Report information extraction.""" self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id) @@ -1216,7 +1303,7 @@ class DailymotionIE(InfoExtractor): video_title = mobj.group(1).decode('utf-8') video_title = sanitize_title(video_title) - mobj = re.search(r'(?im)
.*?(.+?)', webpage) + mobj = re.search(r'(?im)(.+?)', webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract uploader nickname') return @@ -1236,7 +1323,7 @@ class DailymotionIE(InfoExtractor): 'player_url': None, }) except UnavailableVideoError: - self._downloader.trouble(u'ERROR: unable to download video') + self._downloader.trouble(u'\nERROR: unable to download video') class GoogleIE(InfoExtractor): """Information extractor for video.google.com.""" @@ -1346,7 +1433,7 @@ class GoogleIE(InfoExtractor): 'player_url': None, }) except UnavailableVideoError: - self._downloader.trouble(u'ERROR: unable to download video') + self._downloader.trouble(u'\nERROR: unable to download video') class PhotobucketIE(InfoExtractor): @@ -1428,7 +1515,7 @@ class PhotobucketIE(InfoExtractor): 'player_url': None, }) except UnavailableVideoError: - self._downloader.trouble(u'ERROR: unable to download video') + self._downloader.trouble(u'\nERROR: unable to download video') class YahooIE(InfoExtractor): @@ -1586,7 +1673,7 @@ class YahooIE(InfoExtractor): 'player_url': None, }) except UnavailableVideoError: - self._downloader.trouble(u'ERROR: unable to download video') + self._downloader.trouble(u'\nERROR: unable to download video') class GenericIE(InfoExtractor): @@ -1687,7 +1774,7 @@ class GenericIE(InfoExtractor): 'player_url': None, }) except UnavailableVideoError, err: - self._downloader.trouble(u'ERROR: unable to download video') + self._downloader.trouble(u'\nERROR: unable to download video') class YoutubeSearchIE(InfoExtractor): @@ -1702,7 +1789,7 @@ class YoutubeSearchIE(InfoExtractor): def __init__(self, youtube_ie, downloader=None): InfoExtractor.__init__(self, downloader) self._youtube_ie = youtube_ie - + @staticmethod def suitable(url): return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None) @@ -1714,7 +1801,7 @@ class YoutubeSearchIE(InfoExtractor): def _real_initialize(self): self._youtube_ie.initialize() - + def _real_extract(self, query): mobj = re.match(self._VALID_QUERY, query) if mobj is None: @@ -1755,7 +1842,7 @@ class YoutubeSearchIE(InfoExtractor): while True: self.report_download_page(query, pagenum) result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) - request = urllib2.Request(result_url, None, std_headers) + request = urllib2.Request(result_url) try: page = urllib2.urlopen(request).read() except (urllib2.URLError, httplib.HTTPException, socket.error), err: @@ -1793,7 +1880,7 @@ class GoogleSearchIE(InfoExtractor): def __init__(self, google_ie, downloader=None): InfoExtractor.__init__(self, downloader) self._google_ie = google_ie - + @staticmethod def suitable(url): return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None) @@ -1805,7 +1892,7 @@ class GoogleSearchIE(InfoExtractor): def _real_initialize(self): self._google_ie.initialize() - + def _real_extract(self, query): mobj = re.match(self._VALID_QUERY, query) if mobj is None: @@ -1846,7 +1933,7 @@ class GoogleSearchIE(InfoExtractor): while True: self.report_download_page(query, pagenum) result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) - request = urllib2.Request(result_url, None, std_headers) + request = urllib2.Request(result_url) try: page = urllib2.urlopen(request).read() except (urllib2.URLError, httplib.HTTPException, socket.error), err: @@ -1884,7 +1971,7 @@ class YahooSearchIE(InfoExtractor): def __init__(self, yahoo_ie, downloader=None): InfoExtractor.__init__(self, downloader) self._yahoo_ie = yahoo_ie - + @staticmethod def suitable(url): return (re.match(YahooSearchIE._VALID_QUERY, url) is not None) @@ -1896,7 +1983,7 @@ class YahooSearchIE(InfoExtractor): def _real_initialize(self): self._yahoo_ie.initialize() - + def _real_extract(self, query): mobj = re.match(self._VALID_QUERY, query) if mobj is None: @@ -1937,7 +2024,7 @@ class YahooSearchIE(InfoExtractor): while True: self.report_download_page(query, pagenum) result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) - request = urllib2.Request(result_url, None, std_headers) + request = urllib2.Request(result_url) try: page = urllib2.urlopen(request).read() except (urllib2.URLError, httplib.HTTPException, socket.error), err: @@ -1966,7 +2053,7 @@ class YahooSearchIE(InfoExtractor): class YoutubePlaylistIE(InfoExtractor): """Information Extractor for YouTube playlists.""" - _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/)([^&]+).*' + _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/|p/)([^&]+).*' _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en' _VIDEO_INDICATOR = r'/watch\?v=(.+?)&' _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*' @@ -1975,7 +2062,7 @@ class YoutubePlaylistIE(InfoExtractor): def __init__(self, youtube_ie, downloader=None): InfoExtractor.__init__(self, downloader) self._youtube_ie = youtube_ie - + @staticmethod def suitable(url): return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None) @@ -1986,7 +2073,7 @@ class YoutubePlaylistIE(InfoExtractor): def _real_initialize(self): self._youtube_ie.initialize() - + def _real_extract(self, url): # Extract playlist id mobj = re.match(self._VALID_URL, url) @@ -2001,7 +2088,7 @@ class YoutubePlaylistIE(InfoExtractor): while True: self.report_download_page(playlist_id, pagenum) - request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers) + request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum)) try: page = urllib2.urlopen(request).read() except (urllib2.URLError, httplib.HTTPException, socket.error), err: @@ -2040,7 +2127,7 @@ class YoutubeUserIE(InfoExtractor): def __init__(self, youtube_ie, downloader=None): InfoExtractor.__init__(self, downloader) self._youtube_ie = youtube_ie - + @staticmethod def suitable(url): return (re.match(YoutubeUserIE._VALID_URL, url) is not None) @@ -2051,7 +2138,7 @@ class YoutubeUserIE(InfoExtractor): def _real_initialize(self): self._youtube_ie.initialize() - + def _real_extract(self, url): # Extract username mobj = re.match(self._VALID_URL, url) @@ -2137,7 +2224,7 @@ class DepositFilesIE(InfoExtractor): # Retrieve file webpage with 'Free download' button pressed free_download_indication = { 'gateway_result' : '1' } - request = urllib2.Request(url, urllib.urlencode(free_download_indication), std_headers) + request = urllib2.Request(url, urllib.urlencode(free_download_indication)) try: self.report_download_webpage(file_id) webpage = urllib2.urlopen(request).read() @@ -2208,7 +2295,7 @@ class PostProcessor(object): def set_downloader(self, downloader): """Sets the downloader for this PP.""" self._downloader = downloader - + def run(self, information): """Run the PostProcessor. @@ -2228,7 +2315,7 @@ class PostProcessor(object): it was called from. """ return information # by default, do nothing - + ### MAIN PROGRAM ### if __name__ == '__main__': try: @@ -2236,20 +2323,26 @@ if __name__ == '__main__': import getpass import optparse - # Function to update the program file with the latest version from bitbucket.org + # Function to update the program file with the latest version from the repository. def update_self(downloader, filename): # Note: downloader only used for options - if not os.access (filename, os.W_OK): + if not os.access(filename, os.W_OK): sys.exit('ERROR: no write permissions on %s' % filename) downloader.to_screen('Updating to latest stable version...') - latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION' - latest_version = urllib.urlopen(latest_url).read().strip() - prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version - newcontent = urllib.urlopen(prog_url).read() - stream = open(filename, 'w') - stream.write(newcontent) - stream.close() + try: + latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION' + latest_version = urllib.urlopen(latest_url).read().strip() + prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version + newcontent = urllib.urlopen(prog_url).read() + except (IOError, OSError), err: + sys.exit('ERROR: unable to download latest version') + try: + stream = open(filename, 'w') + stream.write(newcontent) + stream.close() + except (IOError, OSError), err: + sys.exit('ERROR: unable to overwrite current version') downloader.to_screen('Updated to version %s' % latest_version) # Parse command line @@ -2275,6 +2368,8 @@ if __name__ == '__main__': dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1) parser.add_option('--playlist-end', dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) + parser.add_option('--dump-user-agent', + action='store_true', dest='dump_user_agent', help='display the current browser identification', default=False) authentication = optparse.OptionGroup(parser, 'Authentication Options') authentication.add_option('-u', '--username', @@ -2309,6 +2404,8 @@ if __name__ == '__main__': action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False) verbosity.add_option('--no-progress', action='store_true', dest='noprogress', help='do not print progress bar', default=False) + verbosity.add_option('--console-title', + action='store_true', dest='consoletitle', help='display progress in console titlebar', default=False) parser.add_option_group(verbosity) filesystem = optparse.OptionGroup(parser, 'Filesystem Options') @@ -2328,6 +2425,8 @@ if __name__ == '__main__': action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False) filesystem.add_option('--cookies', dest='cookiefile', metavar='FILE', help='file to dump cookie jar to') + filesystem.add_option('--no-part', + action='store_true', dest='nopart', help='do not use .part files', default=False) parser.add_option_group(filesystem) (opts, args) = parser.parse_args() @@ -2343,10 +2442,14 @@ if __name__ == '__main__': except (IOError, OSError), err: sys.exit(u'ERROR: unable to open cookie file') + # Dump user agent + if opts.dump_user_agent: + print std_headers['User-Agent'] + sys.exit(0) + # General configuration cookie_processor = urllib2.HTTPCookieProcessor(jar) - urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler())) - urllib2.install_opener(urllib2.build_opener(cookie_processor)) + urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler())) socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words) # Batch file verification @@ -2445,6 +2548,8 @@ if __name__ == '__main__': 'playliststart': opts.playliststart, 'playlistend': opts.playlistend, 'logtostderr': opts.outtmpl == '-', + 'consoletitle': opts.consoletitle, + 'nopart': opts.nopart, }) fd.add_info_extractor(youtube_search_ie) fd.add_info_extractor(youtube_pl_ie)