From c018281550457d55a82db7744e13cd7c377b7143 Mon Sep 17 00:00:00 2001 From: codesparkle Date: Fri, 16 May 2014 16:14:34 +1000 Subject: [PATCH 01/10] Unified format for docstrings: 3x double quote used instead of 3x single quote --- test/helper.py | 4 ++-- youtube_dl/YoutubeDL.py | 18 +++++++++--------- youtube_dl/__init__.py | 2 +- youtube_dl/extractor/channel9.py | 4 ++-- youtube_dl/extractor/ted.py | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/helper.py b/test/helper.py index 230d2bd67..b020caa11 100644 --- a/test/helper.py +++ b/test/helper.py @@ -35,10 +35,10 @@ def try_rm(filename): def report_warning(message): - ''' + """ Print the message to stderr, it will be prefixed with 'WARNING:' If stderr is a tty file the 'WARNING:' will be colored - ''' + """ if sys.stderr.isatty() and os.name != 'nt': _msg_header = u'\033[0;33mWARNING:\033[0m' else: diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index f3666573a..e99d9a74a 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -376,10 +376,10 @@ class YoutubeDL(object): self._download_retcode = 1 def report_warning(self, message): - ''' + """ Print the message to stderr, it will be prefixed with 'WARNING:' If stderr is a tty file the 'WARNING:' will be colored - ''' + """ if self.params.get('logger') is not None: self.params['logger'].warning(message) else: @@ -393,10 +393,10 @@ class YoutubeDL(object): self.to_stderr(warning_message) def report_error(self, message, tb=None): - ''' + """ Do the same as trouble, but prefixes the message with 'ERROR:', colored in red if stderr is a tty file. - ''' + """ if self._err_file.isatty() and os.name != 'nt': _msg_header = '\033[0;31mERROR:\033[0m' else: @@ -487,17 +487,17 @@ class YoutubeDL(object): @staticmethod def add_extra_info(info_dict, extra_info): - '''Set the keys from extra_info in info dict if they are missing''' + """Set the keys from extra_info in info dict if they are missing""" for key, value in extra_info.items(): info_dict.setdefault(key, value) def extract_info(self, url, download=True, ie_key=None, extra_info={}, process=True): - ''' + """ Returns a list with a dictionary for each video we find. If 'download', also downloads the videos. extra_info is a dict containing the extra values to add to each result - ''' + """ if ie_key: ies = [self.get_info_extractor(ie_key)] @@ -1019,7 +1019,7 @@ class YoutubeDL(object): if success: try: self.post_process(filename, info_dict) - except (PostProcessingError) as err: + except PostProcessingError as err: self.report_error('postprocessing: %s' % str(err)) return @@ -1298,7 +1298,7 @@ class YoutubeDL(object): try: return s.encode(self.get_encoding()) except UnicodeEncodeError as err: - err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.' + err.reason += '. Check your system encoding configuration or use the --encoding option.' raise def get_encoding(self): diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 4e657e297..be85de025 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -158,7 +158,7 @@ def parseOpts(overrideArguments=None): return userConf def _format_option_string(option): - ''' ('-o', '--option') -> -o, --format METAVAR''' + """ ('-o', '--option') -> -o, --format METAVAR""" opts = [] diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py index 4f000292b..515701cef 100644 --- a/youtube_dl/extractor/channel9.py +++ b/youtube_dl/extractor/channel9.py @@ -6,13 +6,13 @@ from .common import InfoExtractor from ..utils import ExtractorError class Channel9IE(InfoExtractor): - ''' + """ Common extractor for channel9.msdn.com. The type of provided URL (video or playlist) is determined according to meta Search.PageType from web page HTML rather than URL itself, as it is not always possible to do. - ''' + """ IE_DESC = 'Channel 9' IE_NAME = 'channel9' _VALID_URL = r'https?://(?:www\.)?channel9\.msdn\.com/(?P.+)/?' diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py index d260c91c2..02cac714f 100644 --- a/youtube_dl/extractor/ted.py +++ b/youtube_dl/extractor/ted.py @@ -86,7 +86,7 @@ class TEDIE(SubtitlesInfoExtractor): return self._playlist_videos_info(url, name) def _playlist_videos_info(self, url, name): - '''Returns the videos of the playlist''' + """Returns the videos of the playlist""" webpage = self._download_webpage(url, name, 'Downloading playlist webpage') From 3c71f4bcf0f91dffe99fa492472d9389a099031a Mon Sep 17 00:00:00 2001 From: codesparkle Date: Fri, 16 May 2014 18:08:53 +1000 Subject: [PATCH 02/10] Removed redundant parentheses and replaced expressions of the form x = x + n with x += n --- youtube_dl/aes.py | 6 +++--- youtube_dl/extractor/common.py | 6 +++--- youtube_dl/extractor/dailymotion.py | 2 +- youtube_dl/extractor/francetv.py | 2 +- youtube_dl/extractor/hark.py | 2 +- youtube_dl/extractor/justintv.py | 2 +- youtube_dl/extractor/youtube.py | 2 +- youtube_dl/utils.py | 10 +++++----- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py index e9c5e2152..3e53ba745 100644 --- a/youtube_dl/aes.py +++ b/youtube_dl/aes.py @@ -261,7 +261,7 @@ def xor(data1, data2): return [x^y for x, y in zip(data1, data2)] def rijndael_mul(a, b): - if(a==0 or b==0): + if a==0 or b==0: return 0 return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF] @@ -301,10 +301,10 @@ def shift_rows_inv(data): def inc(data): data = data[:] # copy - for i in range(len(data)-1,-1,-1): + for i in range(len(data)-1, -1, -1): if data[i] == 255: data[i] = 0 else: - data[i] = data[i] + 1 + data[i] += 1 break return data diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index db472aace..6d4646b0b 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -269,7 +269,7 @@ class InfoExtractor(object): msg += u' Visit %s for more details' % blocked_iframe raise ExtractorError(msg, expected=True) - return (content, urlh) + return content, urlh def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True): """ Returns the data of the page as a string """ @@ -399,7 +399,7 @@ class InfoExtractor(object): If there's no info available, return (None, None) """ if self._downloader is None: - return (None, None) + return None, None username = None password = None @@ -420,7 +420,7 @@ class InfoExtractor(object): except (IOError, netrc.NetrcParseError) as err: self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err)) - return (username, password) + return username, password # Helper functions for extracting OpenGraph info @staticmethod diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index 55216201f..7b5a851b8 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -168,7 +168,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err)) return {} info = json.loads(sub_list) - if (info['total'] > 0): + if info['total'] > 0: sub_lang_list = dict((l['language'], l['url']) for l in info['list']) return sub_lang_list self._downloader.report_warning(u'video doesn\'t have subtitles') diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py index 51eb97b2f..ca41872db 100644 --- a/youtube_dl/extractor/francetv.py +++ b/youtube_dl/extractor/francetv.py @@ -161,7 +161,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor): class="francetv-video-player">'''), (r'][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix class BaseHTMLParser(compat_html_parser.HTMLParser): @@ -435,9 +435,9 @@ def sanitize_open(filename, open_mode): if sys.platform == 'win32': import msvcrt msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename) + return sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename stream = open(encodeFilename(filename), open_mode) - return (stream, filename) + return stream, filename except (IOError, OSError) as err: if err.errno in (errno.EACCES,): raise @@ -452,7 +452,7 @@ def sanitize_open(filename, open_mode): else: # An exception here should be caught in the caller stream = open(encodeFilename(filename), open_mode) - return (stream, alt_filename) + return stream, alt_filename def timeconvert(timestr): @@ -604,7 +604,7 @@ class ExtractorError(Exception): if video_id is not None: msg = video_id + ': ' + msg if not expected: - msg = msg + u'; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type youtube-dl -U to update.' + msg += u'; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type youtube-dl -U to update.' super(ExtractorError, self).__init__(msg) self.traceback = tb From 3bf622302758c390f8cceb6890d5a5a46d03f620 Mon Sep 17 00:00:00 2001 From: codesparkle Date: Fri, 16 May 2014 18:09:38 +1000 Subject: [PATCH 03/10] tabs to spaces --- devscripts/gh-pages/sign-versions.py | 16 ++++++++-------- devscripts/transition_helper.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/devscripts/gh-pages/sign-versions.py b/devscripts/gh-pages/sign-versions.py index 8a824df56..40ee17eb6 100755 --- a/devscripts/gh-pages/sign-versions.py +++ b/devscripts/gh-pages/sign-versions.py @@ -11,18 +11,18 @@ except NameError: versions_info = json.load(open('update/versions.json')) if 'signature' in versions_info: - del versions_info['signature'] + del versions_info['signature'] print('Enter the PKCS1 private key, followed by a blank line:') privkey = b'' while True: - try: - line = input() - except EOFError: - break - if line == '': - break - privkey += line.encode('ascii') + b'\n' + try: + line = input() + except EOFError: + break + if line == '': + break + privkey += line.encode('ascii') + b'\n' privkey = rsa.PrivateKey.load_pkcs1(privkey) signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).encode('utf-8'), privkey, 'SHA-256')).decode() diff --git a/devscripts/transition_helper.py b/devscripts/transition_helper.py index d5ca2d4ba..cba875076 100644 --- a/devscripts/transition_helper.py +++ b/devscripts/transition_helper.py @@ -12,9 +12,9 @@ sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorr sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n') try: - raw_input() + raw_input() except NameError: # Python 3 - input() + input() filename = sys.argv[0] From 24331a0fb424138e6319c99ec598ec300e1bc6a4 Mon Sep 17 00:00:00 2001 From: codesparkle Date: Fri, 16 May 2014 18:14:01 +1000 Subject: [PATCH 04/10] Removed unused variables and expressions --- devscripts/gh-pages/update-sites.py | 2 +- youtube_dl/extractor/appletrailers.py | 1 - youtube_dl/extractor/crunchyroll.py | 8 +------- youtube_dl/extractor/mixcloud.py | 2 +- youtube_dl/extractor/mtv.py | 2 +- youtube_dl/extractor/ndtv.py | 2 +- 6 files changed, 5 insertions(+), 12 deletions(-) diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py index 153e15c8a..1905a5b9c 100755 --- a/devscripts/gh-pages/update-sites.py +++ b/devscripts/gh-pages/update-sites.py @@ -21,7 +21,7 @@ def main(): continue elif ie_desc is not None: ie_html += ': {}'.format(ie.IE_DESC) - if ie.working() == False: + if not ie.working(): ie_html += ' (Currently broken)' ie_htmls.append('
  • {}
    • '.format(ie_html)) diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py index dc8657b67..a1a477d65 100644 --- a/youtube_dl/extractor/appletrailers.py +++ b/youtube_dl/extractor/appletrailers.py @@ -119,7 +119,6 @@ class AppleTrailersIE(InfoExtractor): playlist.append({ '_type': 'video', 'id': video_id, - 'title': title, 'formats': formats, 'title': title, 'duration': duration, diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index 026a9177e..bcefe695e 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -76,12 +76,6 @@ class CrunchyrollIE(InfoExtractor): return shaHash + [0] * 12 key = obfuscate_key(id) - class Counter: - __value = iv - def next_value(self): - temp = self.__value - self.__value = inc(self.__value) - return temp decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv)) return zlib.decompress(decrypted_data) @@ -159,7 +153,7 @@ class CrunchyrollIE(InfoExtractor): subtitles = {} for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage): - sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id,\ + sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id, video_id, note='Downloading subtitles for '+sub_name) id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False) iv = self._search_regex(r'([^<]+)', sub_page, 'subtitle_iv', fatal=False) diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py index 5f64e7bd0..1e1f27f0f 100644 --- a/youtube_dl/extractor/mixcloud.py +++ b/youtube_dl/extractor/mixcloud.py @@ -41,7 +41,7 @@ class MixcloudIE(InfoExtractor): self._request_webpage(url, None, False) return url except ExtractorError: - url = None + pass return None diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py index d75241d3f..af5d61cdd 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dl/extractor/mtv.py @@ -204,7 +204,7 @@ class MTVIE(MTVServicesInfoExtractor): m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";', webpage, re.DOTALL) if m_vevo: - vevo_id = m_vevo.group(1); + vevo_id = m_vevo.group(1) self.to_screen('Vevo video detected: %s' % vevo_id) return self.url_result('vevo:%s' % vevo_id, ie='Vevo') diff --git a/youtube_dl/extractor/ndtv.py b/youtube_dl/extractor/ndtv.py index d81df3c10..5b7bee29d 100644 --- a/youtube_dl/extractor/ndtv.py +++ b/youtube_dl/extractor/ndtv.py @@ -31,7 +31,7 @@ class NDTVIE(InfoExtractor): video_url = (u'http://bitcast-b.bitgravity.com/ndtvod/23372/ndtv/%s' % filename) - duration_str = filename = self._search_regex( + duration_str = self._search_regex( r"__duration='([^']+)'", webpage, u'duration', fatal=False) duration = None if duration_str is None else int(duration_str) From c9b32da1eccd2d50ed6549b44c75d651224b05b3 Mon Sep 17 00:00:00 2001 From: codesparkle Date: Fri, 16 May 2014 16:14:34 +1000 Subject: [PATCH 05/10] Unified format for docstrings: 3x double quote used instead of 3x single quote --- test/helper.py | 4 ++-- youtube_dl/YoutubeDL.py | 18 +++++++++--------- youtube_dl/__init__.py | 2 +- youtube_dl/extractor/channel9.py | 4 ++-- youtube_dl/extractor/ted.py | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/helper.py b/test/helper.py index 230d2bd67..b020caa11 100644 --- a/test/helper.py +++ b/test/helper.py @@ -35,10 +35,10 @@ def try_rm(filename): def report_warning(message): - ''' + """ Print the message to stderr, it will be prefixed with 'WARNING:' If stderr is a tty file the 'WARNING:' will be colored - ''' + """ if sys.stderr.isatty() and os.name != 'nt': _msg_header = u'\033[0;33mWARNING:\033[0m' else: diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index f3666573a..e99d9a74a 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -376,10 +376,10 @@ class YoutubeDL(object): self._download_retcode = 1 def report_warning(self, message): - ''' + """ Print the message to stderr, it will be prefixed with 'WARNING:' If stderr is a tty file the 'WARNING:' will be colored - ''' + """ if self.params.get('logger') is not None: self.params['logger'].warning(message) else: @@ -393,10 +393,10 @@ class YoutubeDL(object): self.to_stderr(warning_message) def report_error(self, message, tb=None): - ''' + """ Do the same as trouble, but prefixes the message with 'ERROR:', colored in red if stderr is a tty file. - ''' + """ if self._err_file.isatty() and os.name != 'nt': _msg_header = '\033[0;31mERROR:\033[0m' else: @@ -487,17 +487,17 @@ class YoutubeDL(object): @staticmethod def add_extra_info(info_dict, extra_info): - '''Set the keys from extra_info in info dict if they are missing''' + """Set the keys from extra_info in info dict if they are missing""" for key, value in extra_info.items(): info_dict.setdefault(key, value) def extract_info(self, url, download=True, ie_key=None, extra_info={}, process=True): - ''' + """ Returns a list with a dictionary for each video we find. If 'download', also downloads the videos. extra_info is a dict containing the extra values to add to each result - ''' + """ if ie_key: ies = [self.get_info_extractor(ie_key)] @@ -1019,7 +1019,7 @@ class YoutubeDL(object): if success: try: self.post_process(filename, info_dict) - except (PostProcessingError) as err: + except PostProcessingError as err: self.report_error('postprocessing: %s' % str(err)) return @@ -1298,7 +1298,7 @@ class YoutubeDL(object): try: return s.encode(self.get_encoding()) except UnicodeEncodeError as err: - err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.' + err.reason += '. Check your system encoding configuration or use the --encoding option.' raise def get_encoding(self): diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 4e657e297..be85de025 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -158,7 +158,7 @@ def parseOpts(overrideArguments=None): return userConf def _format_option_string(option): - ''' ('-o', '--option') -> -o, --format METAVAR''' + """ ('-o', '--option') -> -o, --format METAVAR""" opts = [] diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py index 4f000292b..515701cef 100644 --- a/youtube_dl/extractor/channel9.py +++ b/youtube_dl/extractor/channel9.py @@ -6,13 +6,13 @@ from .common import InfoExtractor from ..utils import ExtractorError class Channel9IE(InfoExtractor): - ''' + """ Common extractor for channel9.msdn.com. The type of provided URL (video or playlist) is determined according to meta Search.PageType from web page HTML rather than URL itself, as it is not always possible to do. - ''' + """ IE_DESC = 'Channel 9' IE_NAME = 'channel9' _VALID_URL = r'https?://(?:www\.)?channel9\.msdn\.com/(?P.+)/?' diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py index d260c91c2..02cac714f 100644 --- a/youtube_dl/extractor/ted.py +++ b/youtube_dl/extractor/ted.py @@ -86,7 +86,7 @@ class TEDIE(SubtitlesInfoExtractor): return self._playlist_videos_info(url, name) def _playlist_videos_info(self, url, name): - '''Returns the videos of the playlist''' + """Returns the videos of the playlist""" webpage = self._download_webpage(url, name, 'Downloading playlist webpage') From c370ced80e6510ebdf85104a6dbd21d043d3e366 Mon Sep 17 00:00:00 2001 From: codesparkle Date: Fri, 16 May 2014 18:08:53 +1000 Subject: [PATCH 06/10] Removed redundant parentheses and replaced expressions of the form x = x + n with x += n --- youtube_dl/aes.py | 6 +++--- youtube_dl/extractor/common.py | 6 +++--- youtube_dl/extractor/dailymotion.py | 2 +- youtube_dl/extractor/francetv.py | 2 +- youtube_dl/extractor/hark.py | 2 +- youtube_dl/extractor/justintv.py | 2 +- youtube_dl/extractor/youtube.py | 2 +- youtube_dl/utils.py | 10 +++++----- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py index e9c5e2152..3e53ba745 100644 --- a/youtube_dl/aes.py +++ b/youtube_dl/aes.py @@ -261,7 +261,7 @@ def xor(data1, data2): return [x^y for x, y in zip(data1, data2)] def rijndael_mul(a, b): - if(a==0 or b==0): + if a==0 or b==0: return 0 return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF] @@ -301,10 +301,10 @@ def shift_rows_inv(data): def inc(data): data = data[:] # copy - for i in range(len(data)-1,-1,-1): + for i in range(len(data)-1, -1, -1): if data[i] == 255: data[i] = 0 else: - data[i] = data[i] + 1 + data[i] += 1 break return data diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index db472aace..6d4646b0b 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -269,7 +269,7 @@ class InfoExtractor(object): msg += u' Visit %s for more details' % blocked_iframe raise ExtractorError(msg, expected=True) - return (content, urlh) + return content, urlh def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True): """ Returns the data of the page as a string """ @@ -399,7 +399,7 @@ class InfoExtractor(object): If there's no info available, return (None, None) """ if self._downloader is None: - return (None, None) + return None, None username = None password = None @@ -420,7 +420,7 @@ class InfoExtractor(object): except (IOError, netrc.NetrcParseError) as err: self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err)) - return (username, password) + return username, password # Helper functions for extracting OpenGraph info @staticmethod diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index 55216201f..7b5a851b8 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -168,7 +168,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err)) return {} info = json.loads(sub_list) - if (info['total'] > 0): + if info['total'] > 0: sub_lang_list = dict((l['language'], l['url']) for l in info['list']) return sub_lang_list self._downloader.report_warning(u'video doesn\'t have subtitles') diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py index f3e0f38b7..58aecd941 100644 --- a/youtube_dl/extractor/francetv.py +++ b/youtube_dl/extractor/francetv.py @@ -173,7 +173,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor): class="francetv-video-player">'''), (r'    ][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix class BaseHTMLParser(compat_html_parser.HTMLParser): @@ -435,9 +435,9 @@ def sanitize_open(filename, open_mode): if sys.platform == 'win32': import msvcrt msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename) + return sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename stream = open(encodeFilename(filename), open_mode) - return (stream, filename) + return stream, filename except (IOError, OSError) as err: if err.errno in (errno.EACCES,): raise @@ -452,7 +452,7 @@ def sanitize_open(filename, open_mode): else: # An exception here should be caught in the caller stream = open(encodeFilename(filename), open_mode) - return (stream, alt_filename) + return stream, alt_filename def timeconvert(timestr): @@ -614,7 +614,7 @@ class ExtractorError(Exception): if video_id is not None: msg = video_id + ': ' + msg if not expected: - msg = msg + u'; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type youtube-dl -U to update.' + msg += u'; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type youtube-dl -U to update.' super(ExtractorError, self).__init__(msg) self.traceback = tb From df6662ab8c6724c01ff203f0656b1f95fe83dddf Mon Sep 17 00:00:00 2001 From: codesparkle Date: Fri, 16 May 2014 18:09:38 +1000 Subject: [PATCH 07/10] tabs to spaces --- devscripts/gh-pages/sign-versions.py | 16 ++++++++-------- devscripts/transition_helper.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/devscripts/gh-pages/sign-versions.py b/devscripts/gh-pages/sign-versions.py index 8a824df56..40ee17eb6 100755 --- a/devscripts/gh-pages/sign-versions.py +++ b/devscripts/gh-pages/sign-versions.py @@ -11,18 +11,18 @@ except NameError: versions_info = json.load(open('update/versions.json')) if 'signature' in versions_info: - del versions_info['signature'] + del versions_info['signature'] print('Enter the PKCS1 private key, followed by a blank line:') privkey = b'' while True: - try: - line = input() - except EOFError: - break - if line == '': - break - privkey += line.encode('ascii') + b'\n' + try: + line = input() + except EOFError: + break + if line == '': + break + privkey += line.encode('ascii') + b'\n' privkey = rsa.PrivateKey.load_pkcs1(privkey) signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).encode('utf-8'), privkey, 'SHA-256')).decode() diff --git a/devscripts/transition_helper.py b/devscripts/transition_helper.py index d5ca2d4ba..cba875076 100644 --- a/devscripts/transition_helper.py +++ b/devscripts/transition_helper.py @@ -12,9 +12,9 @@ sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorr sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n') try: - raw_input() + raw_input() except NameError: # Python 3 - input() + input() filename = sys.argv[0] From e594643f335f28c0ffa61610faf71f14a8328a2c Mon Sep 17 00:00:00 2001 From: codesparkle Date: Fri, 16 May 2014 18:14:01 +1000 Subject: [PATCH 08/10] Removed unused variables and expressions --- devscripts/gh-pages/update-sites.py | 2 +- youtube_dl/extractor/appletrailers.py | 1 - youtube_dl/extractor/crunchyroll.py | 8 +------- youtube_dl/extractor/mixcloud.py | 2 +- youtube_dl/extractor/mtv.py | 2 +- youtube_dl/extractor/ndtv.py | 2 +- 6 files changed, 5 insertions(+), 12 deletions(-) diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py index 153e15c8a..1905a5b9c 100755 --- a/devscripts/gh-pages/update-sites.py +++ b/devscripts/gh-pages/update-sites.py @@ -21,7 +21,7 @@ def main(): continue elif ie_desc is not None: ie_html += ': {}'.format(ie.IE_DESC) - if ie.working() == False: + if not ie.working(): ie_html += ' (Currently broken)' ie_htmls.append('
  • {}
    • '.format(ie_html)) diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py index dc8657b67..a1a477d65 100644 --- a/youtube_dl/extractor/appletrailers.py +++ b/youtube_dl/extractor/appletrailers.py @@ -119,7 +119,6 @@ class AppleTrailersIE(InfoExtractor): playlist.append({ '_type': 'video', 'id': video_id, - 'title': title, 'formats': formats, 'title': title, 'duration': duration, diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index 026a9177e..bcefe695e 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -76,12 +76,6 @@ class CrunchyrollIE(InfoExtractor): return shaHash + [0] * 12 key = obfuscate_key(id) - class Counter: - __value = iv - def next_value(self): - temp = self.__value - self.__value = inc(self.__value) - return temp decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv)) return zlib.decompress(decrypted_data) @@ -159,7 +153,7 @@ class CrunchyrollIE(InfoExtractor): subtitles = {} for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage): - sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id,\ + sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id, video_id, note='Downloading subtitles for '+sub_name) id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False) iv = self._search_regex(r'([^<]+)', sub_page, 'subtitle_iv', fatal=False) diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py index 5f64e7bd0..1e1f27f0f 100644 --- a/youtube_dl/extractor/mixcloud.py +++ b/youtube_dl/extractor/mixcloud.py @@ -41,7 +41,7 @@ class MixcloudIE(InfoExtractor): self._request_webpage(url, None, False) return url except ExtractorError: - url = None + pass return None diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py index d75241d3f..af5d61cdd 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dl/extractor/mtv.py @@ -204,7 +204,7 @@ class MTVIE(MTVServicesInfoExtractor): m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";', webpage, re.DOTALL) if m_vevo: - vevo_id = m_vevo.group(1); + vevo_id = m_vevo.group(1) self.to_screen('Vevo video detected: %s' % vevo_id) return self.url_result('vevo:%s' % vevo_id, ie='Vevo') diff --git a/youtube_dl/extractor/ndtv.py b/youtube_dl/extractor/ndtv.py index d81df3c10..5b7bee29d 100644 --- a/youtube_dl/extractor/ndtv.py +++ b/youtube_dl/extractor/ndtv.py @@ -31,7 +31,7 @@ class NDTVIE(InfoExtractor): video_url = (u'http://bitcast-b.bitgravity.com/ndtvod/23372/ndtv/%s' % filename) - duration_str = filename = self._search_regex( + duration_str = self._search_regex( r"__duration='([^']+)'", webpage, u'duration', fatal=False) duration = None if duration_str is None else int(duration_str) From 8d5cb73ea47031062a55c466172175485d0b6258 Mon Sep 17 00:00:00 2001 From: codesparkle Date: Sat, 17 May 2014 01:43:03 +1000 Subject: [PATCH 09/10] test_download works for photobucket after this change --- youtube_dl/extractor/photobucket.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/photobucket.py b/youtube_dl/extractor/photobucket.py index a59953497..5337dea56 100644 --- a/youtube_dl/extractor/photobucket.py +++ b/youtube_dl/extractor/photobucket.py @@ -1,10 +1,10 @@ from __future__ import unicode_literals -import datetime import json import re from .common import InfoExtractor +from utils import compat_urllib_parse class PhotobucketIE(InfoExtractor): @@ -14,6 +14,7 @@ class PhotobucketIE(InfoExtractor): 'file': 'zpsc0c3b9fa.mp4', 'md5': '7dabfb92b0a31f6c16cebc0f8e60ff99', 'info_dict': { + 'timestamp': 1367669341, 'upload_date': '20130504', 'uploader': 'rachaneronas', 'title': 'Tired of Link Building? Try BacklinkMyDomain.com!', @@ -32,11 +33,12 @@ class PhotobucketIE(InfoExtractor): info_json = self._search_regex(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (.*?)\);', webpage, 'info json') info = json.loads(info_json) + url = compat_urllib_parse.unquote(self._html_search_regex('file=(.+\.mp4)', info['linkcodes']['html'], 'url')) return { 'id': video_id, - 'url': info['downloadUrl'], + 'url': url, 'uploader': info['username'], - 'upload_date': datetime.date.fromtimestamp(info['creationDate']).strftime('%Y%m%d'), + 'timestamp': info['creationDate'], 'title': info['title'], 'ext': video_extension, 'thumbnail': info['thumbUrl'], From c047e092e99461e1d8c07de56ef0ebe720fde989 Mon Sep 17 00:00:00 2001 From: codesparkle Date: Sat, 17 May 2014 02:51:39 +1000 Subject: [PATCH 10/10] Revert "[bliptv] Switch extraction to RSS (Closes #2920)" This reverts commit 481efc84a8a9e7bfabf5e9ee357ae7b47eeb9127. --- youtube_dl/extractor/bliptv.py | 186 ++++++++++++++------------------- 1 file changed, 81 insertions(+), 105 deletions(-) diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py index d4da08991..a26001bb3 100644 --- a/youtube_dl/extractor/bliptv.py +++ b/youtube_dl/extractor/bliptv.py @@ -1,124 +1,102 @@ from __future__ import unicode_literals +import datetime import re from .common import InfoExtractor from .subtitles import SubtitlesInfoExtractor from ..utils import ( - compat_urllib_request, - unescapeHTML, - parse_iso8601, - compat_urlparse, - clean_html, compat_str, + compat_urllib_request, + + unescapeHTML, ) class BlipTVIE(SubtitlesInfoExtractor): - _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P\d+)|((?:play/|api\.swf#)(?P[\da-zA-Z]+)))' + """Information extractor for blip.tv""" - _TESTS = [ - { - 'url': 'http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352', - 'md5': 'c6934ad0b6acf2bd920720ec888eb812', - 'info_dict': { - 'id': '5779306', - 'ext': 'mov', - 'title': 'CBR EXCLUSIVE: "Gotham City Imposters" Bats VS Jokerz Short 3', - 'description': 'md5:9bc31f227219cde65e47eeec8d2dc596', - 'timestamp': 1323138843, - 'upload_date': '20111206', - 'uploader': 'cbr', - 'uploader_id': '679425', - 'duration': 81, - } - }, - { - # https://github.com/rg3/youtube-dl/pull/2274 - 'note': 'Video with subtitles', - 'url': 'http://blip.tv/play/h6Uag5OEVgI.html', - 'md5': '309f9d25b820b086ca163ffac8031806', - 'info_dict': { - 'id': '6586561', - 'ext': 'mp4', - 'title': 'Red vs. Blue Season 11 Episode 1', - 'description': 'One-Zero-One', - 'timestamp': 1371261608, - 'upload_date': '20130615', - 'uploader': 'redvsblue', - 'uploader_id': '792887', - 'duration': 279, - } + _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/((.+/)|(play/)|(api\.swf#))(?P.+)$' + + _TESTS = [{ + 'url': 'http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352', + 'md5': 'c6934ad0b6acf2bd920720ec888eb812', + 'info_dict': { + 'id': '5779306', + 'ext': 'mov', + 'upload_date': '20111205', + 'description': 'md5:9bc31f227219cde65e47eeec8d2dc596', + 'uploader': 'Comic Book Resources - CBR TV', + 'title': 'CBR EXCLUSIVE: "Gotham City Imposters" Bats VS Jokerz Short 3', } - ] + }, { + # https://github.com/rg3/youtube-dl/pull/2274 + 'note': 'Video with subtitles', + 'url': 'http://blip.tv/play/h6Uag5OEVgI.html', + 'md5': '309f9d25b820b086ca163ffac8031806', + 'info_dict': { + 'id': '6586561', + 'ext': 'mp4', + 'uploader': 'Red vs. Blue', + 'description': 'One-Zero-One', + 'upload_date': '20130614', + 'title': 'Red vs. Blue Season 11 Episode 1', + } + }] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - lookup_id = mobj.group('lookup_id') + presumptive_id = mobj.group('presumptive_id') # See https://github.com/rg3/youtube-dl/issues/857 - if lookup_id: - info_page = self._download_webpage( - 'http://blip.tv/play/%s.x?p=1' % lookup_id, lookup_id, 'Resolving lookup id') - video_id = self._search_regex(r'data-episode-id="([0-9]+)', info_page, 'video_id') + embed_mobj = re.match(r'https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)([a-zA-Z0-9]+)', url) + if embed_mobj: + info_url = 'http://blip.tv/play/%s.x?p=1' % embed_mobj.group(1) + info_page = self._download_webpage(info_url, embed_mobj.group(1)) + video_id = self._search_regex( + r'data-episode-id="([0-9]+)', info_page, 'video_id') + return self.url_result('http://blip.tv/a/a-' + video_id, 'BlipTV') + + cchar = '&' if '?' in url else '?' + json_url = url + cchar + 'skin=json&version=2&no_wrap=1' + request = compat_urllib_request.Request(json_url) + request.add_header('User-Agent', 'iTunes/10.6.1') + + json_data = self._download_json(request, video_id=presumptive_id) + + if 'Post' in json_data: + data = json_data['Post'] else: - video_id = mobj.group('id') + data = json_data - rss = self._download_xml('http://blip.tv/rss/flash/%s' % video_id, video_id, 'Downloading video RSS') - - def blip(s): - return '{http://blip.tv/dtd/blip/1.0}%s' % s - - def media(s): - return '{http://search.yahoo.com/mrss/}%s' % s - - def itunes(s): - return '{http://www.itunes.com/dtds/podcast-1.0.dtd}%s' % s - - item = rss.find('channel/item') - - video_id = item.find(blip('item_id')).text - title = item.find('./title').text - description = clean_html(compat_str(item.find(blip('puredescription')).text)) - timestamp = parse_iso8601(item.find(blip('datestamp')).text) - uploader = item.find(blip('user')).text - uploader_id = item.find(blip('userid')).text - duration = int(item.find(blip('runtime')).text) - media_thumbnail = item.find(media('thumbnail')) - thumbnail = media_thumbnail.get('url') if media_thumbnail is not None else item.find(itunes('image')).text - categories = [category.text for category in item.findall('category')] - - formats = [] + video_id = compat_str(data['item_id']) + upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d') subtitles = {} - - media_group = item.find(media('group')) - for media_content in media_group.findall(media('content')): - url = media_content.get('url') - role = media_content.get(blip('role')) - msg = self._download_webpage( - url + '?showplayer=20140425131715&referrer=http://blip.tv&mask=7&skin=flashvars&view=url', - video_id, 'Resolving URL for %s' % role) - real_url = compat_urlparse.parse_qs(msg)['message'][0] - - media_type = media_content.get('type') - if media_type == 'text/srt' or url.endswith('.srt'): - LANGS = { - 'english': 'en', - } - lang = role.rpartition('-')[-1].strip().lower() - langcode = LANGS.get(lang, lang) - subtitles[langcode] = url - elif media_type.startswith('video/'): + formats = [] + if 'additionalMedia' in data: + for f in data['additionalMedia']: + if f.get('file_type_srt') == 1: + LANGS = { + 'english': 'en', + } + lang = f['role'].rpartition('-')[-1].strip().lower() + langcode = LANGS.get(lang, lang) + subtitles[langcode] = f['url'] + continue + if not int(f['media_width']): # filter m3u8 + continue formats.append({ - 'url': real_url, - 'format_id': role, - 'format_note': media_type, - 'vcodec': media_content.get(blip('vcodec')), - 'acodec': media_content.get(blip('acodec')), - 'filesize': media_content.get('filesize'), - 'width': int(media_content.get('width')), - 'height': int(media_content.get('height')), + 'url': f['url'], + 'format_id': f['role'], + 'width': int(f['media_width']), + 'height': int(f['media_height']), }) + else: + formats.append({ + 'url': data['media']['url'], + 'width': int(data['media']['width']), + 'height': int(data['media']['height']), + }) self._sort_formats(formats) # subtitles @@ -129,14 +107,12 @@ class BlipTVIE(SubtitlesInfoExtractor): return { 'id': video_id, - 'title': title, - 'description': description, - 'timestamp': timestamp, - 'uploader': uploader, - 'uploader_id': uploader_id, - 'duration': duration, - 'thumbnail': thumbnail, - 'categories': categories, + 'uploader': data['display_name'], + 'upload_date': upload_date, + 'title': data['title'], + 'thumbnail': data['thumbnailUrl'], + 'description': data['description'], + 'user_agent': 'iTunes/10.6.1', 'formats': formats, 'subtitles': video_subtitles, }