Akkariin/youtube-dl
1
0
Fork 0
mirror of https://github.com/l1ving/youtube-dl synced 2025-03-12 01:07:22 +08:00
Code Issues Releases Wiki Activity

Prepare strings for Python 2.5-3 transition

Browse Source
This commit is contained in:
Philipp Hagemeister 2012-11-27 20:26:56 +01:00
parent efe4b51b2d
commit 51065d3efd
7 changed files with 490 additions and 482 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
test/test_utils.py
View File

@ -1,15 +1,22 @@
# -*- coding: utf-8 -*-
# Various small unit tests # Various small unit tests
import sys
import unittest import unittest
# Allow direct execution
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
#from youtube_dl.utils import htmlentity_transform #from youtube_dl.utils import htmlentity_transform
from youtube_dl.utils import timeconvert from youtube_dl.utils import timeconvert
from youtube_dl.utils import sanitize_filename from youtube_dl.utils import sanitize_filename
from youtube_dl.utils import unescapeHTML from youtube_dl.utils import unescapeHTML
from youtube_dl.utils import orderedSet from youtube_dl.utils import orderedSet
if sys.version < (3,0):
_compat_str = lambda b: b.decode('unicode-escape')
else:
_compat_str = lambda s: s
class TestUtil(unittest.TestCase): class TestUtil(unittest.TestCase):
def test_timeconvert(self): def test_timeconvert(self):
@ -17,56 +24,59 @@ class TestUtil(unittest.TestCase):
self.assertTrue(timeconvert('bougrg') is None) self.assertTrue(timeconvert('bougrg') is None)
def test_sanitize_filename(self): def test_sanitize_filename(self):
self.assertEqual(sanitize_filename(u'abc'), u'abc') self.assertEqual(sanitize_filename('abc'), 'abc')
self.assertEqual(sanitize_filename(u'abc_d-e'), u'abc_d-e') self.assertEqual(sanitize_filename('abc_d-e'), 'abc_d-e')
self.assertEqual(sanitize_filename(u'123'), u'123') self.assertEqual(sanitize_filename('123'), '123')
self.assertEqual(u'abc_de', sanitize_filename(u'abc/de')) self.assertEqual('abc_de', sanitize_filename('abc/de'))
self.assertFalse(u'/' in sanitize_filename(u'abc/de///')) self.assertFalse('/' in sanitize_filename('abc/de///'))
self.assertEqual(u'abc_de', sanitize_filename(u'abc/<>\\*|de')) self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de'))
self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|')) self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|'))
self.assertEqual(u'yes no', sanitize_filename(u'yes? no')) self.assertEqual('yes no', sanitize_filename('yes? no'))
self.assertEqual(u'this - that', sanitize_filename(u'this: that')) self.assertEqual('this - that', sanitize_filename('this: that'))
self.assertEqual(sanitize_filename(u'AT&T'), u'AT&T') self.assertEqual(sanitize_filename('AT&T'), 'AT&T')
self.assertEqual(sanitize_filename(u'ä'), u'ä') aumlaut = _compat_str('\xe4')
self.assertEqual(sanitize_filename(u'кириллица'), u'кириллица') self.assertEqual(sanitize_filename(aumlaut), aumlaut)
tests = _compat_str('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430')
self.assertEqual(sanitize_filename(tests), tests)
forbidden = u'"\0\\/' forbidden = '"\0\\/'
for fc in forbidden: for fc in forbidden:
for fbc in forbidden: for fbc in forbidden:
self.assertTrue(fbc not in sanitize_filename(fc)) self.assertTrue(fbc not in sanitize_filename(fc))
def test_sanitize_filename_restricted(self): def test_sanitize_filename_restricted(self):
self.assertEqual(sanitize_filename(u'abc', restricted=True), u'abc') self.assertEqual(sanitize_filename('abc', restricted=True), 'abc')
self.assertEqual(sanitize_filename(u'abc_d-e', restricted=True), u'abc_d-e') self.assertEqual(sanitize_filename('abc_d-e', restricted=True), 'abc_d-e')
self.assertEqual(sanitize_filename(u'123', restricted=True), u'123') self.assertEqual(sanitize_filename('123', restricted=True), '123')
self.assertEqual(u'abc_de', sanitize_filename(u'abc/de', restricted=True)) self.assertEqual('abc_de', sanitize_filename('abc/de', restricted=True))
self.assertFalse(u'/' in sanitize_filename(u'abc/de///', restricted=True)) self.assertFalse('/' in sanitize_filename('abc/de///', restricted=True))
self.assertEqual(u'abc_de', sanitize_filename(u'abc/<>\\*|de', restricted=True)) self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de', restricted=True))
self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|', restricted=True)) self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|', restricted=True))
self.assertEqual(u'yes_no', sanitize_filename(u'yes? no', restricted=True)) self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True))
self.assertEqual(u'this_-_that', sanitize_filename(u'this: that', restricted=True)) self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True))
self.assertEqual(sanitize_filename(u'aäb中国的c', restricted=True), u'a_b_c') tests =_compat_str('a\xe4b\u4e2d\u56fd\u7684c')
self.assertTrue(sanitize_filename(u'ö', restricted=True) != u'') # No empty filename self.assertEqual(sanitize_filename(tests, restricted=True), 'a_b_c')
self.assertTrue(sanitize_filename(_compat_str('\xf6'), restricted=True) != '') # No empty filename
forbidden = u'"\0\\/&!: \'\t\n' forbidden = '"\0\\/&!: \'\t\n'
for fc in forbidden: for fc in forbidden:
for fbc in forbidden: for fbc in forbidden:
self.assertTrue(fbc not in sanitize_filename(fc, restricted=True)) self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
# Handle a common case more neatly # Handle a common case more neatly
self.assertEqual(sanitize_filename(u'大声带 - Song', restricted=True), u'Song') self.assertEqual(sanitize_filename(_compat_str('\u5927\u58f0\u5e26 - Song'), restricted=True), 'Song')
self.assertEqual(sanitize_filename(u'总统: Speech', restricted=True), u'Speech') self.assertEqual(sanitize_filename(_compat_str('\u603b\u7edf: Speech'), restricted=True), 'Speech')
# .. but make sure the file name is never empty # .. but make sure the file name is never empty
self.assertTrue(sanitize_filename(u'-', restricted=True) != u'') self.assertTrue(sanitize_filename('-', restricted=True) != '')
self.assertTrue(sanitize_filename(u':', restricted=True) != u'') self.assertTrue(sanitize_filename(':', restricted=True) != '')
def test_ordered_set(self): def test_ordered_set(self):
self.assertEqual(orderedSet([1,1,2,3,4,4,5,6,7,3,5]), [1,2,3,4,5,6,7]) self.assertEqual(orderedSet([1,1,2,3,4,4,5,6,7,3,5]), [1,2,3,4,5,6,7])
@ -76,4 +86,7 @@ class TestUtil(unittest.TestCase):
self.assertEqual(orderedSet([135,1,1,1]), [135,1]) self.assertEqual(orderedSet([135,1,1,1]), [135,1])
def test_unescape_html(self): def test_unescape_html(self):
self.assertEqual(unescapeHTML(u"%20;"), u"%20;") self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;'))
if __name__ == '__main__':
unittest.main()

103
youtube_dl/FileDownloader.py
View File

@ -1,5 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*-
import httplib import httplib
import math import math
@ -97,7 +96,7 @@ class FileDownloader(object):
self.params = params self.params = params
if '%(stitle)s' in self.params['outtmpl']: if '%(stitle)s' in self.params['outtmpl']:
self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') self.to_stderr(u('WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.'))
@staticmethod @staticmethod
def format_bytes(bytes): def format_bytes(bytes):
@ -175,9 +174,9 @@ class FileDownloader(object):
def to_screen(self, message, skip_eol=False): def to_screen(self, message, skip_eol=False):
"""Print message to stdout if not in quiet mode.""" """Print message to stdout if not in quiet mode."""
assert type(message) == type(u'') assert type(message) == type(u(''))
if not self.params.get('quiet', False): if not self.params.get('quiet', False):
terminator = [u'\n', u''][skip_eol] terminator = [u('\n'), u('')][skip_eol]
output = message + terminator output = message + terminator
if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
output = output.encode(preferredencoding(), 'ignore') output = output.encode(preferredencoding(), 'ignore')
@ -186,8 +185,8 @@ class FileDownloader(object):
def to_stderr(self, message): def to_stderr(self, message):
"""Print message to stderr.""" """Print message to stderr."""
assert type(message) == type(u'') assert type(message) == type(u(''))
sys.stderr.write((message + u'\n').encode(preferredencoding())) sys.stderr.write((message + u('\n')).encode(preferredencoding()))
def to_cons_title(self, message): def to_cons_title(self, message):
"""Set console/terminal window title to message.""" """Set console/terminal window title to message."""
@ -232,14 +231,14 @@ class FileDownloader(object):
def temp_name(self, filename): def temp_name(self, filename):
"""Returns a temporary filename for the given filename.""" """Returns a temporary filename for the given filename."""
if self.params.get('nopart', False) or filename == u'-' or \ if self.params.get('nopart', False) or filename == u('-') or \
(os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
return filename return filename
return filename + u'.part' return filename + u('.part')
def undo_temp_name(self, filename): def undo_temp_name(self, filename):
if filename.endswith(u'.part'): if filename.endswith(u('.part')):
return filename[:-len(u'.part')] return filename[:-len(u('.part'))]
return filename return filename
def try_rename(self, old_filename, new_filename): def try_rename(self, old_filename, new_filename):
@ -249,7 +248,7 @@ class FileDownloader(object):
os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
except (IOError, OSError): except (IOError, OSError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
self.trouble(u'ERROR: unable to rename file') self.trouble(u('ERROR: unable to rename file'))
def try_utime(self, filename, last_modified_hdr): def try_utime(self, filename, last_modified_hdr):
"""Try to set the last-modified time of the given file.""" """Try to set the last-modified time of the given file."""
@ -271,55 +270,55 @@ class FileDownloader(object):
def report_writedescription(self, descfn): def report_writedescription(self, descfn):
""" Report that the description file is being written """ """ Report that the description file is being written """
self.to_screen(u'[info] Writing video description to: ' + descfn) self.to_screen(u('[info] Writing video description to: ') + descfn)
def report_writesubtitles(self, srtfn): def report_writesubtitles(self, srtfn):
""" Report that the subtitles file is being written """ """ Report that the subtitles file is being written """
self.to_screen(u'[info] Writing video subtitles to: ' + srtfn) self.to_screen(u('[info] Writing video subtitles to: ') + srtfn)
def report_writeinfojson(self, infofn): def report_writeinfojson(self, infofn):
""" Report that the metadata file has been written """ """ Report that the metadata file has been written """
self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) self.to_screen(u('[info] Video description metadata as JSON to: ') + infofn)
def report_destination(self, filename): def report_destination(self, filename):
"""Report destination filename.""" """Report destination filename."""
self.to_screen(u'[download] Destination: ' + filename) self.to_screen(u('[download] Destination: ') + filename)
def report_progress(self, percent_str, data_len_str, speed_str, eta_str): def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
"""Report download progress.""" """Report download progress."""
if self.params.get('noprogress', False): if self.params.get('noprogress', False):
return return
self.to_screen(u'\r[download] %s of %s at %s ETA %s' % self.to_screen(u('\r[download] %s of %s at %s ETA %s') %
(percent_str, data_len_str, speed_str, eta_str), skip_eol=True) (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' % self.to_cons_title(u('youtube-dl - %s of %s at %s ETA %s') %
(percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip())) (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
def report_resuming_byte(self, resume_len): def report_resuming_byte(self, resume_len):
"""Report attempt to resume at given byte.""" """Report attempt to resume at given byte."""
self.to_screen(u'[download] Resuming download at byte %s' % resume_len) self.to_screen(u('[download] Resuming download at byte %s') % resume_len)
def report_retry(self, count, retries): def report_retry(self, count, retries):
"""Report retry in case of HTTP error 5xx""" """Report retry in case of HTTP error 5xx"""
self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) self.to_screen(u('[download] Got server HTTP error. Retrying (attempt %d of %d)...') % (count, retries))
def report_file_already_downloaded(self, file_name): def report_file_already_downloaded(self, file_name):
"""Report file has already been fully downloaded.""" """Report file has already been fully downloaded."""
try: try:
self.to_screen(u'[download] %s has already been downloaded' % file_name) self.to_screen(u('[download] %s has already been downloaded') % file_name)
except (UnicodeEncodeError): except (UnicodeEncodeError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
self.to_screen(u'[download] The file has already been downloaded') self.to_screen(u('[download] The file has already been downloaded'))
def report_unable_to_resume(self): def report_unable_to_resume(self):
"""Report it was impossible to resume download.""" """Report it was impossible to resume download."""
self.to_screen(u'[download] Unable to resume') self.to_screen(u('[download] Unable to resume'))
def report_finish(self): def report_finish(self):
"""Report download finished.""" """Report download finished."""
if self.params.get('noprogress', False): if self.params.get('noprogress', False):
self.to_screen(u'[download] Download completed') self.to_screen(u('[download] Download completed'))
else: else:
self.to_screen(u'') self.to_screen(u(''))
def increment_downloads(self): def increment_downloads(self):
"""Increment the ordinal that assigns a number to each file.""" """Increment the ordinal that assigns a number to each file."""
@ -330,14 +329,14 @@ class FileDownloader(object):
try: try:
template_dict = dict(info_dict) template_dict = dict(info_dict)
template_dict['epoch'] = int(time.time()) template_dict['epoch'] = int(time.time())
template_dict['autonumber'] = u'%05d' % self._num_downloads template_dict['autonumber'] = u('%05d') % self._num_downloads
template_dict = dict((k, sanitize_filename(u(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items()) template_dict = dict((k, sanitize_filename(u(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
filename = self.params['outtmpl'] % template_dict filename = self.params['outtmpl'] % template_dict
return filename return filename
except (ValueError, KeyError): except (ValueError, KeyError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
self.trouble(u'ERROR: invalid system charset or erroneous output template') self.trouble(u('ERROR: invalid system charset or erroneous output template'))
return None return None
def _match_entry(self, info_dict): def _match_entry(self, info_dict):
@ -348,12 +347,12 @@ class FileDownloader(object):
if matchtitle: if matchtitle:
matchtitle = matchtitle.decode('utf8') matchtitle = matchtitle.decode('utf8')
if not re.search(matchtitle, title, re.IGNORECASE): if not re.search(matchtitle, title, re.IGNORECASE):
return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' return u('[download] "') + title + '" title did not match pattern "' + matchtitle + '"'
rejecttitle = self.params.get('rejecttitle', False) rejecttitle = self.params.get('rejecttitle', False)
if rejecttitle: if rejecttitle:
rejecttitle = rejecttitle.decode('utf8') rejecttitle = rejecttitle.decode('utf8')
if re.search(rejecttitle, title, re.IGNORECASE): if re.search(rejecttitle, title, re.IGNORECASE):
return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' return u('"') + title + '" title matched reject pattern "' + rejecttitle + '"'
return None return None
def process_info(self, info_dict): def process_info(self, info_dict):
@ -364,7 +363,7 @@ class FileDownloader(object):
reason = self._match_entry(info_dict) reason = self._match_entry(info_dict)
if reason is not None: if reason is not None:
self.to_screen(u'[download] ' + reason) self.to_screen(u('[download] ') + reason)
return return
max_downloads = self.params.get('max_downloads') max_downloads = self.params.get('max_downloads')
@ -401,12 +400,12 @@ class FileDownloader(object):
os.makedirs(dn) os.makedirs(dn)
except (OSError, IOError): except (OSError, IOError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
self.trouble(u'ERROR: unable to create directory ' + u(err)) self.trouble(u('ERROR: unable to create directory ') + u(err))
return return
if self.params.get('writedescription', False): if self.params.get('writedescription', False):
try: try:
descfn = filename + u'.description' descfn = filename + u('.description')
self.report_writedescription(descfn) self.report_writedescription(descfn)
descfile = open(encodeFilename(descfn), 'wb') descfile = open(encodeFilename(descfn), 'wb')
try: try:
@ -414,14 +413,14 @@ class FileDownloader(object):
finally: finally:
descfile.close() descfile.close()
except (OSError, IOError): except (OSError, IOError):
self.trouble(u'ERROR: Cannot write description file ' + descfn) self.trouble(u('ERROR: Cannot write description file ') + descfn)
return return
if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']: if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
# subtitles download errors are already managed as troubles in relevant IE # subtitles download errors are already managed as troubles in relevant IE
# that way it will silently go on when used with unsupporting IE # that way it will silently go on when used with unsupporting IE
try: try:
srtfn = filename.rsplit('.', 1)[0] + u'.srt' srtfn = filename.rsplit('.', 1)[0] + u('.srt')
self.report_writesubtitles(srtfn) self.report_writesubtitles(srtfn)
srtfile = open(encodeFilename(srtfn), 'wb') srtfile = open(encodeFilename(srtfn), 'wb')
try: try:
@ -429,16 +428,16 @@ class FileDownloader(object):
finally: finally:
srtfile.close() srtfile.close()
except (OSError, IOError): except (OSError, IOError):
self.trouble(u'ERROR: Cannot write subtitles file ' + descfn) self.trouble(u('ERROR: Cannot write subtitles file ') + descfn)
return return
if self.params.get('writeinfojson', False): if self.params.get('writeinfojson', False):
infofn = filename + u'.info.json' infofn = filename + u('.info.json')
self.report_writeinfojson(infofn) self.report_writeinfojson(infofn)
try: try:
json.dump json.dump
except (NameError,AttributeError): except (NameError,AttributeError):
self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.') self.trouble(u('ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.'))
return return
try: try:
infof = open(encodeFilename(infofn), 'wb') infof = open(encodeFilename(infofn), 'wb')
@ -448,7 +447,7 @@ class FileDownloader(object):
finally: finally:
infof.close() infof.close()
except (OSError, IOError): except (OSError, IOError):
self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn) self.trouble(u('ERROR: Cannot write metadata to JSON file ') + infofn)
return return
if not self.params.get('skip_download', False): if not self.params.get('skip_download', False):
@ -462,11 +461,11 @@ class FileDownloader(object):
raise UnavailableVideoError raise UnavailableVideoError
except (urllib2.URLError, httplib.HTTPException, socket.error): except (urllib2.URLError, httplib.HTTPException, socket.error):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
self.trouble(u'ERROR: unable to download video data: %s' % str(err)) self.trouble(u('ERROR: unable to download video data: %s') % str(err))
return return
except (ContentTooShortError, ): except (ContentTooShortError, ):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) self.trouble(u('ERROR: content too short (expected %s bytes and served %s)') % (err.expected, err.downloaded))
return return
if success: if success:
@ -474,7 +473,7 @@ class FileDownloader(object):
self.post_process(filename, info_dict) self.post_process(filename, info_dict)
except (PostProcessingError): except (PostProcessingError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
self.trouble(u'ERROR: postprocessing: %s' % str(err)) self.trouble(u('ERROR: postprocessing: %s') % str(err))
return return
def download(self, url_list): def download(self, url_list):
@ -500,13 +499,13 @@ class FileDownloader(object):
self.increment_downloads() self.increment_downloads()
self.process_info(video) self.process_info(video)
except UnavailableVideoError: except UnavailableVideoError:
self.trouble(u'\nERROR: unable to download video') self.trouble(u('\nERROR: unable to download video'))
# Suitable InfoExtractor had been found; go to next URL # Suitable InfoExtractor had been found; go to next URL
break break
if not suitable_found: if not suitable_found:
self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) self.trouble(u('ERROR: no suitable InfoExtractor: %s') % url)
return self._download_retcode return self._download_retcode
@ -527,7 +526,7 @@ class FileDownloader(object):
try: try:
subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT) subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
except (OSError, IOError): except (OSError, IOError):
self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run') self.trouble(u('ERROR: RTMP download detected but "rtmpdump" could not be run'))
return False return False
# Download using rtmpdump. rtmpdump returns exit code 2 when # Download using rtmpdump. rtmpdump returns exit code 2 when
@ -541,11 +540,11 @@ class FileDownloader(object):
shell_quote = lambda args: ' '.join(map(pipes.quote, args)) shell_quote = lambda args: ' '.join(map(pipes.quote, args))
except ImportError: except ImportError:
shell_quote = repr shell_quote = repr
self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args)) self.to_screen(u('[debug] rtmpdump command line: ') + shell_quote(args))
retval = subprocess.call(args) retval = subprocess.call(args)
while retval == 2 or retval == 1: while retval == 2 or retval == 1:
prevsize = os.path.getsize(encodeFilename(tmpfilename)) prevsize = os.path.getsize(encodeFilename(tmpfilename))
self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) self.to_screen(u('\r[rtmpdump] %s bytes') % prevsize, skip_eol=True)
time.sleep(5.0) # This seems to be needed time.sleep(5.0) # This seems to be needed
retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
cursize = os.path.getsize(encodeFilename(tmpfilename)) cursize = os.path.getsize(encodeFilename(tmpfilename))
@ -553,15 +552,15 @@ class FileDownloader(object):
break break
# Some rtmp streams seem abort after ~ 99.8%. Don't complain for those # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
if prevsize == cursize and retval == 2 and cursize > 1024: if prevsize == cursize and retval == 2 and cursize > 1024:
self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.') self.to_screen(u('\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.'))
retval = 0 retval = 0
break break
if retval == 0: if retval == 0:
self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename))) self.to_screen(u('\r[rtmpdump] %s bytes') % os.path.getsize(encodeFilename(tmpfilename)))
self.try_rename(tmpfilename, filename) self.try_rename(tmpfilename, filename)
return True return True
else: else:
self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval) self.trouble(u('\nERROR: rtmpdump exited with code %d') % retval)
return False return False
def _do_download(self, filename, info_dict): def _do_download(self, filename, info_dict):
@ -649,7 +648,7 @@ class FileDownloader(object):
self.report_retry(count, retries) self.report_retry(count, retries)
if count > retries: if count > retries:
self.trouble(u'ERROR: giving up after %s retries' % retries) self.trouble(u('ERROR: giving up after %s retries') % retries)
return False return False
data_len = data.info().get('Content-length', None) data_len = data.info().get('Content-length', None)
@ -677,13 +676,13 @@ class FileDownloader(object):
self.report_destination(filename) self.report_destination(filename)
except (OSError, IOError): except (OSError, IOError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
self.trouble(u'ERROR: unable to open for writing: %s' % str(err)) self.trouble(u('ERROR: unable to open for writing: %s') % str(err))
return False return False
try: try:
stream.write(data_block) stream.write(data_block)
except (IOError, OSError): except (IOError, OSError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
self.trouble(u'\nERROR: unable to write data: %s' % str(err)) self.trouble(u('\nERROR: unable to write data: %s') % str(err))
return False return False
if not self.params.get('noresizebuffer', False): if not self.params.get('noresizebuffer', False):
block_size = self.best_block_size(after - before, len(data_block)) block_size = self.best_block_size(after - before, len(data_block))
@ -701,7 +700,7 @@ class FileDownloader(object):
self.slow_down(start, byte_counter - resume_len) self.slow_down(start, byte_counter - resume_len)
if stream is None: if stream is None:
self.trouble(u'\nERROR: Did not get any data blocks') self.trouble(u('\nERROR: Did not get any data blocks'))
return False return False
stream.close() stream.close()
self.report_finish() self.report_finish()

687
youtube_dl/InfoExtractors.py
View File

File diff suppressed because it is too large Load Diff

15
youtube_dl/PostProcessor.py
View File

@ -1,5 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*-
import os import os
import subprocess import subprocess
@ -119,7 +118,7 @@ class FFmpegExtractAudioPP(PostProcessor):
filecodec = self.get_audio_codec(path) filecodec = self.get_audio_codec(path)
if filecodec is None: if filecodec is None:
self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe') self._downloader.to_stderr(u('WARNING: unable to obtain file audio codec with ffprobe'))
return None return None
more_opts = [] more_opts = []
@ -167,17 +166,17 @@ class FFmpegExtractAudioPP(PostProcessor):
extension = 'wav' extension = 'wav'
more_opts += ['-f', 'wav'] more_opts += ['-f', 'wav']
prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups prefix, sep, ext = path.rpartition(u('.')) # not os.path.splitext, since the latter does not work on unicode in all setups
new_path = prefix + sep + extension new_path = prefix + sep + extension
self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path) self._downloader.to_screen(u('[') + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
try: try:
self.run_ffmpeg(path, new_path, acodec, more_opts) self.run_ffmpeg(path, new_path, acodec, more_opts)
except: except:
etype,e,tb = sys.exc_info() etype,e,tb = sys.exc_info()
if isinstance(e, AudioConversionError): if isinstance(e, AudioConversionError):
self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message) self._downloader.to_stderr(u('ERROR: audio conversion failed: ') + e.message)
else: else:
self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg')) self._downloader.to_stderr(u('ERROR: error running ') + (self._exes['avconv'] and 'avconv' or 'ffmpeg'))
return None return None
# Try to update the date time for extracted audio file. # Try to update the date time for extracted audio file.
@ -185,13 +184,13 @@ class FFmpegExtractAudioPP(PostProcessor):
try: try:
os.utime(encodeFilename(new_path), (time.time(), information['filetime'])) os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
except: except:
self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file') self._downloader.to_stderr(u('WARNING: Cannot update utime of audio file'))
if not self._keepvideo: if not self._keepvideo:
try: try:
os.remove(encodeFilename(path)) os.remove(encodeFilename(path))
except (IOError, OSError): except (IOError, OSError):
self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file') self._downloader.to_stderr(u('WARNING: Unable to remove downloaded video file'))
return None return None
information['filepath'] = new_path information['filepath'] = new_path

68
youtube_dl/__init__.py
View File

@ -53,12 +53,12 @@ def updateSelf(downloader, filename):
if not os.access(filename, os.W_OK): if not os.access(filename, os.W_OK):
sys.exit('ERROR: no write permissions on %s' % filename) sys.exit('ERROR: no write permissions on %s' % filename)
downloader.to_screen(u'Updating to latest version...') downloader.to_screen(u('Updating to latest version...'))
urlv = urllib2.urlopen(UPDATE_URL_VERSION) urlv = urllib2.urlopen(UPDATE_URL_VERSION)
newversion = urlv.read().strip() newversion = urlv.read().strip()
if newversion == __version__: if newversion == __version__:
downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') downloader.to_screen(u('youtube-dl is up-to-date (') + __version__ + ')')
return return
urlv.close() urlv.close()
@ -110,7 +110,7 @@ del "%s"
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
sys.exit('ERROR: unable to overwrite current version') sys.exit('ERROR: unable to overwrite current version')
downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.') downloader.to_screen(u('Updated youtube-dl. Restart youtube-dl to use the new version.'))
def parseOpts(): def parseOpts():
def _readOptions(filename_bytes): def _readOptions(filename_bytes):
@ -392,14 +392,14 @@ def _real_main():
jar.load() jar.load()
except (IOError, OSError): except (IOError, OSError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
sys.exit(u'ERROR: unable to open cookie file') sys.exit(u('ERROR: unable to open cookie file'))
# Set user agent # Set user agent
if opts.user_agent is not None: if opts.user_agent is not None:
std_headers['User-Agent'] = opts.user_agent std_headers['User-Agent'] = opts.user_agent
# Dump user agent # Dump user agent
if opts.dump_user_agent: if opts.dump_user_agent:
print std_headers['User-Agent'] print(std_headers['User-Agent'])
sys.exit(0) sys.exit(0)
# Batch file verification # Batch file verification
@ -414,7 +414,7 @@ def _real_main():
batchurls = [x.strip() for x in batchurls] batchurls = [x.strip() for x in batchurls]
batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)] batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
except IOError: except IOError:
sys.exit(u'ERROR: batch file could not be read') sys.exit(u('ERROR: batch file could not be read'))
all_urls = batchurls + args all_urls = batchurls + args
all_urls = map(lambda url: url.strip(), all_urls) all_urls = map(lambda url: url.strip(), all_urls)
@ -433,57 +433,57 @@ def _real_main():
matchedUrls = filter(lambda url: ie.suitable(url), all_urls) matchedUrls = filter(lambda url: ie.suitable(url), all_urls)
all_urls = filter(lambda url: url not in matchedUrls, all_urls) all_urls = filter(lambda url: url not in matchedUrls, all_urls)
for mu in matchedUrls: for mu in matchedUrls:
print(u' ' + mu) print(u(' ') + mu)
sys.exit(0) sys.exit(0)
# Conflicting, missing and erroneous options # Conflicting, missing and erroneous options
if opts.usenetrc and (opts.username is not None or opts.password is not None): if opts.usenetrc and (opts.username is not None or opts.password is not None):
parser.error(u'using .netrc conflicts with giving username/password') parser.error(u('using .netrc conflicts with giving username/password'))
if opts.password is not None and opts.username is None: if opts.password is not None and opts.username is None:
parser.error(u'account username missing') parser.error(u('account username missing'))
if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid): if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
parser.error(u'using output template conflicts with using title, video ID or auto number') parser.error(u('using output template conflicts with using title, video ID or auto number'))
if opts.usetitle and opts.useid: if opts.usetitle and opts.useid:
parser.error(u'using title conflicts with using video ID') parser.error(u('using title conflicts with using video ID'))
if opts.username is not None and opts.password is None: if opts.username is not None and opts.password is None:
opts.password = getpass.getpass(u'Type account password and press return:') opts.password = getpass.getpass(u('Type account password and press return:'))
if opts.ratelimit is not None: if opts.ratelimit is not None:
numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
if numeric_limit is None: if numeric_limit is None:
parser.error(u'invalid rate limit specified') parser.error(u('invalid rate limit specified'))
opts.ratelimit = numeric_limit opts.ratelimit = numeric_limit
if opts.retries is not None: if opts.retries is not None:
try: try:
opts.retries = int(opts.retries) opts.retries = int(opts.retries)
except (TypeError, ValueError): except (TypeError, ValueError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
parser.error(u'invalid retry count specified') parser.error(u('invalid retry count specified'))
if opts.buffersize is not None: if opts.buffersize is not None:
numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize) numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
if numeric_buffersize is None: if numeric_buffersize is None:
parser.error(u'invalid buffer size specified') parser.error(u('invalid buffer size specified'))
opts.buffersize = numeric_buffersize opts.buffersize = numeric_buffersize
try: try:
opts.playliststart = int(opts.playliststart) opts.playliststart = int(opts.playliststart)
if opts.playliststart <= 0: if opts.playliststart <= 0:
raise ValueError(u'Playlist start must be positive') raise ValueError(u('Playlist start must be positive'))
except (TypeError, ValueError): except (TypeError, ValueError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
parser.error(u'invalid playlist start number specified') parser.error(u('invalid playlist start number specified'))
try: try:
opts.playlistend = int(opts.playlistend) opts.playlistend = int(opts.playlistend)
if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart): if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
raise ValueError(u'Playlist end must be greater than playlist start') raise ValueError(u('Playlist end must be greater than playlist start'))
except (TypeError, ValueError): except (TypeError, ValueError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
parser.error(u'invalid playlist end number specified') parser.error(u('invalid playlist end number specified'))
if opts.extractaudio: if opts.extractaudio:
if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']: if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']:
parser.error(u'invalid audio format specified') parser.error(u('invalid audio format specified'))
if opts.audioquality: if opts.audioquality:
opts.audioquality = opts.audioquality.strip('k').strip('K') opts.audioquality = opts.audioquality.strip('k').strip('K')
if not opts.audioquality.isdigit(): if not opts.audioquality.isdigit():
parser.error(u'invalid audio quality specified') parser.error(u('invalid audio quality specified'))
# File downloader # File downloader
fd = FileDownloader({ fd = FileDownloader({
@ -503,13 +503,13 @@ def _real_main():
'format_limit': opts.format_limit, 'format_limit': opts.format_limit,
'listformats': opts.listformats, 'listformats': opts.listformats,
'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding())) 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s') or (opts.format == '-1' and opts.usetitle and u('%(title)s-%(id)s-%(format)s.%(ext)s'))
or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s') or (opts.format == '-1' and u('%(id)s-%(format)s.%(ext)s'))
or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s') or (opts.usetitle and opts.autonumber and u('%(autonumber)s-%(title)s-%(id)s.%(ext)s'))
or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s') or (opts.usetitle and u('%(title)s-%(id)s.%(ext)s'))
or (opts.useid and u'%(id)s.%(ext)s') or (opts.useid and u('%(id)s.%(ext)s'))
or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s') or (opts.autonumber and u('%(autonumber)s-%(id)s.%(ext)s'))
or u'%(id)s.%(ext)s'), or u('%(id)s.%(ext)s')),
'restrictfilenames': opts.restrictfilenames, 'restrictfilenames': opts.restrictfilenames,
'ignoreerrors': opts.ignoreerrors, 'ignoreerrors': opts.ignoreerrors,
'ratelimit': opts.ratelimit, 'ratelimit': opts.ratelimit,
@ -537,7 +537,7 @@ def _real_main():
}) })
if opts.verbose: if opts.verbose:
fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies)) fd.to_screen(u('[debug] Proxy map: ') + str(proxy_handler.proxies))
for extractor in extractors: for extractor in extractors:
fd.add_info_extractor(extractor) fd.add_info_extractor(extractor)
@ -553,14 +553,14 @@ def _real_main():
# Maybe do nothing # Maybe do nothing
if len(all_urls) < 1: if len(all_urls) < 1:
if not opts.update_self: if not opts.update_self:
parser.error(u'you must provide at least one URL') parser.error(u('you must provide at least one URL'))
else: else:
sys.exit() sys.exit()
try: try:
retcode = fd.download(all_urls) retcode = fd.download(all_urls)
except MaxDownloadsReached: except MaxDownloadsReached:
fd.to_screen(u'--max-download limit reached, aborting.') fd.to_screen(u('--max-download limit reached, aborting.'))
retcode = 101 retcode = 101
# Dump cookie jar if requested # Dump cookie jar if requested
@ -569,7 +569,7 @@ def _real_main():
jar.save() jar.save()
except (IOError, OSError): except (IOError, OSError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
sys.exit(u'ERROR: unable to save cookie jar') sys.exit(u('ERROR: unable to save cookie jar'))
sys.exit(retcode) sys.exit(retcode)
@ -579,6 +579,6 @@ def main():
except DownloadError: except DownloadError:
sys.exit(1) sys.exit(1)
except SameFileError: except SameFileError:
sys.exit(u'ERROR: fixed output name but more than one file to download') sys.exit(u('ERROR: fixed output name but more than one file to download'))
except KeyboardInterrupt: except KeyboardInterrupt:
sys.exit(u'\nERROR: Interrupted by user') sys.exit(u('\nERROR: Interrupted by user'))

1
youtube_dl/__main__.py
View File

@ -1,5 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*-
import __init__ import __init__

21
youtube_dl/utils.py
View File

@ -1,5 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*-
import gzip import gzip
import htmlentitydefs import htmlentitydefs
@ -39,7 +38,7 @@ def preferredencoding():
""" """
try: try:
pref = locale.getpreferredencoding() pref = locale.getpreferredencoding()
u'TEST'.encode(pref) u('TEST').encode(pref)
except: except:
pref = 'UTF-8' pref = 'UTF-8'
@ -61,15 +60,15 @@ def htmlentity_transform(matchobj):
mobj = re.match(ur'(?u)#(x?\d+)', entity) mobj = re.match(ur'(?u)#(x?\d+)', entity)
if mobj is not None: if mobj is not None:
numstr = mobj.group(1) numstr = mobj.group(1)
if numstr.startswith(u'x'): if numstr.startswith(u('x')):
base = 16 base = 16
numstr = u'0%s' % numstr numstr = u('0%s') % numstr
else: else:
base = 10 base = 10
return unichr(int(numstr, base)) return unichr(int(numstr, base))
# Unknown entity in name, return its literal representation # Unknown entity in name, return its literal representation
return (u'&%s;' % entity) return (u('&%s;') % entity)
HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
class IDParser(HTMLParser.HTMLParser): class IDParser(HTMLParser.HTMLParser):
@ -170,7 +169,7 @@ def sanitize_open(filename, open_mode):
It returns the tuple (stream, definitive_file_name). It returns the tuple (stream, definitive_file_name).
""" """
try: try:
if filename == u'-': if filename == u('-'):
if sys.platform == 'win32': if sys.platform == 'win32':
import msvcrt import msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
@ -180,7 +179,7 @@ def sanitize_open(filename, open_mode):
except (IOError, OSError): except (IOError, OSError):
_, err, _ = sys.exc_info() _, err, _ = sys.exc_info()
# In case of error, try to remove win32 forbidden chars # In case of error, try to remove win32 forbidden chars
filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename) filename = re.sub(ur'[/<>:"\|\?\*]', u('#'), filename)
# An exception here should be caught in the caller # An exception here should be caught in the caller
stream = open(encodeFilename(filename), open_mode) stream = open(encodeFilename(filename), open_mode)
@ -214,7 +213,7 @@ def sanitize_filename(s, restricted=False):
return '_' return '_'
return char return char
result = u''.join(map(replace_insane, s)) result = u('').join(map(replace_insane, s))
while '__' in result: while '__' in result:
result = result.replace('__', '_') result = result.replace('__', '_')
result = result.strip('_') result = result.strip('_')
@ -237,7 +236,7 @@ def unescapeHTML(s):
""" """
@param s a string @param s a string
""" """
assert type(s) == type(u'') assert type(s) == type(u(''))
result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s) result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s)
return result return result
@ -247,10 +246,10 @@ def encodeFilename(s):
@param s The name of the file @param s The name of the file
""" """
assert type(s) == type(u'') assert type(s) == type(u(''))
if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5: if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
# Pass u'' directly to use Unicode APIs on Windows 2000 and up # Pass u('') directly to use Unicode APIs on Windows 2000 and up
# (Detecting Windows NT 4 is tricky because 'major >= 4' would # (Detecting Windows NT 4 is tricky because 'major >= 4' would
# match Windows 9x series as well. Besides, NT 4 is obsolete.) # match Windows 9x series as well. Besides, NT 4 is obsolete.)
return s return s