From f11554092b419baa919875432fe6ebc1f22f5307 Mon Sep 17 00:00:00 2001 From: Tjark Saul Date: Fri, 17 Apr 2015 09:21:54 +0200 Subject: [PATCH 0001/1424] [Lecture2Go] Add new extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/lecture2go.py | 33 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 youtube_dl/extractor/lecture2go.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index bbf3be41d..3d6e981b2 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -249,6 +249,7 @@ from .krasview import KrasViewIE from .ku6 import Ku6IE from .la7 import LA7IE from .laola1tv import Laola1TvIE +from .lecture2go import Lecture2GoIE from .letv import ( LetvIE, LetvTvIE, diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py new file mode 100644 index 000000000..9cf28e31c --- /dev/null +++ b/youtube_dl/extractor/lecture2go.py @@ -0,0 +1,33 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + + +class Lecture2GoIE(InfoExtractor): + _VALID_URL = r'https?://lecture2go.uni-hamburg.de/veranstaltungen/-/v/(?P[0-9]+)' + _TEST = { + 'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473', + 'md5': 'a9e76f83b3ef58019c4b7dbc35f406c1', + 'info_dict': { + 'id': '17473', + 'ext': 'mp4', + 'url': 'https://fms1.rrz.uni-hamburg.de/abo/64.050_FrankHeitmann_2015-04-13_14-35.mp4', + 'title': '2 - Endliche Automaten und reguläre Sprachen' + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + title = self._html_search_regex(r'(.*?)', webpage, 'title') + video_url = self._search_regex(r'b.isFirefox..a.useHTML5\).b.setOption.a,"src","(.*.mp4)"\).else', webpage, 'video_url') + creator = self._html_search_regex(r'
(.*)
', webpage, 'creator') + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'creator': creator + } From 8dab1e9072037daa9c6cab3da4a5dbd4daaae4c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 3 May 2015 09:56:03 +0600 Subject: [PATCH 0002/1424] [rutv] Recognize live streams (#5584) --- youtube_dl/extractor/rutv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py index ef766237b..169f7c032 100644 --- a/youtube_dl/extractor/rutv.py +++ b/youtube_dl/extractor/rutv.py @@ -181,12 +181,15 @@ class RUTVIE(InfoExtractor): self._sort_formats(formats) + is_live = video_type == 'live' + return { 'id': video_id, - 'title': title, + 'title': self._live_title(title) if is_live else title, 'description': description, 'thumbnail': thumbnail, 'view_count': view_count, 'duration': duration, 'formats': formats, + 'is_live': is_live, } From d0fd305023b37f7776485679a74e422eade26c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 3 May 2015 10:00:34 +0600 Subject: [PATCH 0003/1424] [rutv] Add test for #5584 --- youtube_dl/extractor/rutv.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py index 169f7c032..1ec2c86e5 100644 --- a/youtube_dl/extractor/rutv.py +++ b/youtube_dl/extractor/rutv.py @@ -84,11 +84,20 @@ class RUTVIE(InfoExtractor): 'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ', 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c', }, + 'skip': 'Translation has finished', + }, + { + 'url': 'http://live.russia.tv/index/index/channel_id/3', + 'info_dict': { + 'id': '21', + 'ext': 'mp4', + 'title': 're:^Россия 24. Прямой эфир [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', + 'is_live': True, + }, 'params': { - # rtmp download + # m3u8 download 'skip_download': True, }, - 'skip': 'Translation has finished', }, ] From 233c1c0e76d64c9e13dc8968bfd8a014c49e66a8 Mon Sep 17 00:00:00 2001 From: Antti Ajanki Date: Sun, 3 May 2015 11:04:14 +0300 Subject: [PATCH 0004/1424] [downloader/f4m] Fragment filenames must be sanitized because the fragment was written to a file with a sanitized name by http_dl.download() --- youtube_dl/downloader/f4m.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py index b1a858c45..3cb07e15f 100644 --- a/youtube_dl/downloader/f4m.py +++ b/youtube_dl/downloader/f4m.py @@ -396,18 +396,19 @@ class F4mFD(FileDownloader): success = http_dl.download(frag_filename, {'url': url}) if not success: return False - with open(frag_filename, 'rb') as down: - down_data = down.read() - reader = FlvReader(down_data) - while True: - _, box_type, box_data = reader.read_box_info() - if box_type == b'mdat': - dest_stream.write(box_data) - break + (down, frag_sanitized) = sanitize_open(frag_filename, 'rb') + down_data = down.read() + down.close() + reader = FlvReader(down_data) + while True: + _, box_type, box_data = reader.read_box_info() + if box_type == b'mdat': + dest_stream.write(box_data) + break if live: - os.remove(frag_filename) + os.remove(encodeFilename(frag_sanitized)) else: - frags_filenames.append(frag_filename) + frags_filenames.append(frag_sanitized) except (compat_urllib_error.HTTPError, ) as err: if live and (err.code == 404 or err.code == 410): # We didn't keep up with the live window. Continue @@ -430,7 +431,7 @@ class F4mFD(FileDownloader): elapsed = time.time() - start self.try_rename(tmpfilename, filename) for frag_file in frags_filenames: - os.remove(frag_file) + os.remove(encodeFilename(frag_file)) fsize = os.path.getsize(encodeFilename(filename)) self._hook_progress({ From 5477ca82395545e577afb269b910336cd98de5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sun, 3 May 2015 16:59:14 +0200 Subject: [PATCH 0005/1424] [dailymotion] Use https urls The video url still redirects to an http url, but it doesn't explicitly contain the video id. --- youtube_dl/extractor/dailymotion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index 7615ecd4b..aa595af20 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -85,7 +85,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) - url = 'http://www.dailymotion.com/video/%s' % video_id + url = 'https://www.dailymotion.com/video/%s' % video_id # Retrieve video webpage to extract further information request = self._build_request(url) @@ -110,7 +110,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor): if mobj is not None: video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1) - embed_url = 'http://www.dailymotion.com/embed/video/%s' % video_id + embed_url = 'https://www.dailymotion.com/embed/video/%s' % video_id embed_request = self._build_request(embed_url) embed_page = self._download_webpage( embed_request, video_id, 'Downloading embed page') From 1748d67aea66b6d63bc203ada1036110e7fd2402 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Mon, 4 May 2015 01:11:23 +0800 Subject: [PATCH 0006/1424] [lifenews] Fix view count and comment count --- youtube_dl/extractor/lifenews.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py index 1dfe7f77f..3c00ed333 100644 --- a/youtube_dl/extractor/lifenews.py +++ b/youtube_dl/extractor/lifenews.py @@ -47,9 +47,9 @@ class LifeNewsIE(InfoExtractor): description = self._og_search_description(webpage) view_count = self._html_search_regex( - r'
(\d+)
', webpage, 'view count', fatal=False) + r'
\s*(\d+)\s*
', webpage, 'view count', fatal=False) comment_count = self._html_search_regex( - r'
\s*(\d+)', webpage, 'comment count', fatal=False) + r'
\s*\s*(\d+)\s*', webpage, 'comment count', fatal=False) upload_date = self._html_search_regex( r'
\s*

([^<]+)

', + webpage, 'description', fatal=False) + return { + '_type': 'url_transparent', + 'ie_key': 'NDREmbedBase', + 'url': 'ndr:%s' % video_id, + 'display_id': display_id, + 'description': description, } - } -class NDREmbedBaseIE(NDRBaseIE): +class NDREmbedBaseIE(InfoExtractor): + IE_NAME = 'ndr:embed:base' + _VALID_URL = r'(?:ndr:(?P[\da-z]+)|https?://www\.ndr\.de/(?P[\da-z]+)-ppjson\.json)' + _TESTS = [{ + 'url': 'ndr:soundcheck3366', + 'only_matching': True, + }, { + 'url': 'http://www.ndr.de/soundcheck3366-ppjson.json', + 'only_matching': True, + }] def _real_extract(self, url): - video_id = self._match_id(url) - json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, fatal=False) - if not json_data: - raise ExtractorError('No media links available for %s' % video_id) - return self.extract_video_info(json_data['playlist'], video_id) + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') or mobj.group('id_s') + + ppjson = self._download_json( + 'http://www.ndr.de/%s-ppjson.json' % video_id, video_id) + + playlist = ppjson['playlist'] + + formats = [] + quality_key = qualities(('xs', 's', 'm', 'l', 'xl')) + + for format_id, f in playlist.items(): + src = f.get('src') + if not src: + continue + ext = determine_ext(src, None) + if ext == 'f4m': + formats.extend(self._extract_f4m_formats( + src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, f4m_id='hds')) + elif ext == 'm3u8': + formats.extend(self._extract_m3u8_formats( + src, video_id, m3u8_id='hls', entry_protocol='m3u8_native')) + else: + quality = f.get('quality') + ff = { + 'url': src, + 'format_id': quality or format_id, + 'quality': quality_key(quality), + } + type_ = f.get('type') + if type_ and type_.split('/')[0] == 'audio': + ff['vcodec'] = 'none' + ff['ext'] = ext or 'mp3' + formats.append(ff) + self._sort_formats(formats) + + config = playlist['config'] + + live = playlist.get('config', {}).get('streamType') in ['httpVideoLive', 'httpAudioLive'] + title = config['title'] + if live: + title = self._live_title(title) + uploader = ppjson.get('config', {}).get('branding') + upload_date = ppjson.get('config', {}).get('publicationDate') + duration = int_or_none(config.get('duration')) + + thumbnails = [{ + 'id': thumbnail.get('quality') or thumbnail_id, + 'url': thumbnail['src'], + 'preference': quality_key(thumbnail.get('quality')), + } for thumbnail_id, thumbnail in config.get('poster', {}).items() if thumbnail.get('src')] + + return { + 'id': video_id, + 'title': title, + 'is_live': live, + 'uploader': uploader if uploader != '-' else None, + 'upload_date': upload_date[0:8] if upload_date else None, + 'duration': duration, + 'thumbnails': thumbnails, + 'formats': formats, + } class NDREmbedIE(NDREmbedBaseIE): IE_NAME = 'ndr:embed' - _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P[a-z0-9]+)-(?:player|externalPlayer)\.html' - - _TEST = { + _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P[\da-z]+)-(?:player|externalPlayer)\.html' + _TESTS = [{ 'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html', - 'md5': 'cb63be60cd6f9dd75218803146d8dc67', + 'md5': '8b9306142fe65bbdefb5ce24edb6b0a9', 'info_dict': { 'id': 'ndraktuell28488', 'ext': 'mp4', 'title': 'Norddeutschland begrüßt Flüchtlinge', + 'is_live': False, + 'uploader': 'ndrtv', + 'upload_date': '20150907', 'duration': 132, - } - } + }, + }, { + 'url': 'http://www.ndr.de/ndr2/events/soundcheck/soundcheck3366-player.html', + 'md5': '002085c44bae38802d94ae5802a36e78', + 'info_dict': { + 'id': 'soundcheck3366', + 'ext': 'mp4', + 'title': 'Ella Henderson braucht Vergleiche nicht zu scheuen', + 'is_live': False, + 'uploader': 'ndr2', + 'upload_date': '20150912', + 'duration': 3554, + }, + 'params': { + 'skip_download': True, + }, + }, { + 'url': 'http://www.ndr.de/info/audio51535-player.html', + 'md5': 'bb3cd38e24fbcc866d13b50ca59307b8', + 'info_dict': { + 'id': 'audio51535', + 'ext': 'mp3', + 'title': 'La Valette entgeht der Hinrichtung', + 'is_live': False, + 'uploader': 'ndrinfo', + 'upload_date': '20140729', + 'duration': 884, + }, + 'params': { + 'skip_download': True, + }, + }, { + 'url': 'http://www.ndr.de/fernsehen/sendungen/visite/visite11010-externalPlayer.html', + 'md5': 'ae57f80511c1e1f2fd0d0d3d31aeae7c', + 'info_dict': { + 'id': 'visite11010', + 'ext': 'mp4', + 'title': 'Visite - die ganze Sendung', + 'is_live': False, + 'uploader': 'ndrtv', + 'upload_date': '20150902', + 'duration': 3525, + }, + 'params': { + 'skip_download': True, + }, + }, { + # httpVideoLive + 'url': 'http://www.ndr.de/fernsehen/livestream/livestream217-externalPlayer.html', + 'info_dict': { + 'id': 'livestream217', + 'ext': 'flv', + 'title': 're:^NDR Fernsehen Niedersachsen \d{4}-\d{2}-\d{2} \d{2}:\d{2}$', + 'is_live': True, + 'upload_date': '20150910', + }, + 'params': { + 'skip_download': True, + }, + }, { + 'url': 'http://www.ndr.de/ndrkultur/audio255020-player.html', + 'only_matching': True, + }, { + 'url': 'http://www.ndr.de/fernsehen/sendungen/nordtour/nordtour7124-player.html', + 'only_matching': True, + }, { + 'url': 'http://www.ndr.de/kultur/film/videos/videoimport10424-player.html', + 'only_matching': True, + }, { + 'url': 'http://www.ndr.de/fernsehen/sendungen/hamburg_journal/hamj43006-player.html', + 'only_matching': True, + }, { + 'url': 'http://www.ndr.de/fernsehen/sendungen/weltbilder/weltbilder4518-player.html', + 'only_matching': True, + }, { + 'url': 'http://www.ndr.de/fernsehen/doku952-player.html', + 'only_matching': True, + }] class NJoyEmbedIE(NDREmbedBaseIE): - IE_NAME = 'N-JOY:embed' - _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P[a-z0-9]+)-(?:player|externalPlayer)\.html' - - _TEST = { - 'url': 'http://www.n-joy.de/entertainment/film/portraet374-player_image-832d9b79-fa8a-4026-92e2-e0fd99deb2f9_theme-n-joy.html', - 'md5': 'cb63be60cd6f9dd75218803146d8dc67', + IE_NAME = 'njoy:embed' + _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html' + _TESTS = [{ + # httpVideo + 'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html', + 'md5': '8483cbfe2320bd4d28a349d62d88bd74', 'info_dict': { - 'id': 'portraet374', + 'id': 'doku948', 'ext': 'mp4', - 'title': 'Viviane Andereggen - "Schuld um Schuld"', - 'duration': 129, - } - } + 'title': 'Zehn Jahre Reeperbahn Festival - die Doku', + 'is_live': False, + 'upload_date': '20150807', + 'duration': 1011, + }, + }, { + # httpAudio + 'url': 'http://www.n-joy.de/news_wissen/stefanrichter100-player_image-d5e938b1-f21a-4b9a-86b8-aaba8bca3a13_theme-n-joy.html', + 'md5': 'd989f80f28ac954430f7b8a48197188a', + 'info_dict': { + 'id': 'stefanrichter100', + 'ext': 'mp3', + 'title': 'Interview mit einem Augenzeugen', + 'is_live': False, + 'uploader': 'njoy', + 'upload_date': '20150909', + 'duration': 140, + }, + 'params': { + 'skip_download': True, + }, + }, { + # httpAudioLive, no explicit ext + 'url': 'http://www.n-joy.de/news_wissen/webradioweltweit100-player_image-3fec0484-2244-4565-8fb8-ed25fd28b173_theme-n-joy.html', + 'info_dict': { + 'id': 'webradioweltweit100', + 'ext': 'mp3', + 'title': 're:^N-JOY Weltweit \d{4}-\d{2}-\d{2} \d{2}:\d{2}$', + 'is_live': True, + 'uploader': 'njoy', + 'upload_date': '20150810', + }, + 'params': { + 'skip_download': True, + }, + }, { + 'url': 'http://www.n-joy.de/musik/dockville882-player_image-3905259e-0803-4764-ac72-8b7de077d80a_theme-n-joy.html', + 'only_matching': True, + }, { + 'url': 'http://www.n-joy.de/radio/sendungen/morningshow/urlaubsfotos190-player_image-066a5df1-5c95-49ec-a323-941d848718db_theme-n-joy.html', + 'only_matching': True, + }, { + 'url': 'http://www.n-joy.de/entertainment/comedy/krudetv290-player_image-ab261bfe-51bf-4bf3-87ba-c5122ee35b3d_theme-n-joy.html', + 'only_matching': True, + }] From 82c18e2a53a14cf88bc15a1d0b0f4342d2dc5db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 13 Sep 2015 19:18:57 +0600 Subject: [PATCH 1339/1424] [nowness] Simplify --- youtube_dl/extractor/__init__.py | 2 +- youtube_dl/extractor/nowness.py | 107 ++++++++++++++++--------------- 2 files changed, 57 insertions(+), 52 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 99da52d96..74b7df463 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -408,7 +408,7 @@ from .novamov import NovaMovIE from .nowness import ( NownessIE, NownessPlaylistIE, - NownessSerieIE, + NownessSeriesIE, ) from .nowtv import NowTVIE from .nowvideo import NowVideoIE diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py index 9a0e46917..9fb916c18 100644 --- a/youtube_dl/extractor/nowness.py +++ b/youtube_dl/extractor/nowness.py @@ -4,11 +4,14 @@ from __future__ import unicode_literals from .brightcove import BrightcoveIE from .common import InfoExtractor from ..utils import ExtractorError -from ..compat import compat_urllib_request +from ..compat import ( + compat_str, + compat_urllib_request, +) class NownessBaseIE(InfoExtractor): - def extract_url_result(self, post): + def _extract_url_result(self, post): if post['type'] == 'video': for media in post['media']: if media['type'] == 'video': @@ -18,7 +21,7 @@ class NownessBaseIE(InfoExtractor): player_code = self._download_webpage( 'http://www.nowness.com/iframe?id=%s' % video_id, video_id, note='Downloading player JavaScript', - errnote='Player download failed') + errnote='Unable to download player JavaScript') bc_url = BrightcoveIE._extract_brightcove_url(player_code) if bc_url is None: raise ExtractorError('Could not find player definition') @@ -32,50 +35,46 @@ class NownessBaseIE(InfoExtractor): # return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique') pass - def api_request(self, url, request_path): + def _api_request(self, url, request_path): display_id = self._match_id(url) - - lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us' - request = compat_urllib_request.Request('http://api.nowness.com/api/' + request_path % display_id, headers={ - 'X-Nowness-Language': lang, - }) - json_data = self._download_json(request, display_id) - return display_id, json_data + request = compat_urllib_request.Request( + 'http://api.nowness.com/api/' + request_path % display_id, + headers={ + 'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us', + }) + return display_id, self._download_json(request, display_id) class NownessIE(NownessBaseIE): IE_NAME = 'nowness' _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(?:story|(?:series|category)/[^/]+)/(?P[^/]+?)(?:$|[?#])' - _TESTS = [ - { - 'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation', - 'md5': '068bc0202558c2e391924cb8cc470676', - 'info_dict': { - 'id': '2520295746001', - 'ext': 'mp4', - 'title': 'Candor: The Art of Gesticulation', - 'description': 'Candor: The Art of Gesticulation', - 'thumbnail': 're:^https?://.*\.jpg', - 'uploader': 'Nowness', - } - }, - { - 'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr', - 'md5': 'e79cf125e387216f86b2e0a5b5c63aa3', - 'info_dict': { - 'id': '3716354522001', - 'ext': 'mp4', - 'title': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR', - 'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR', - 'thumbnail': 're:^https?://.*\.jpg', - 'uploader': 'Nowness', - } - }, - ] + _TESTS = [{ + 'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation', + 'md5': '068bc0202558c2e391924cb8cc470676', + 'info_dict': { + 'id': '2520295746001', + 'ext': 'mp4', + 'title': 'Candor: The Art of Gesticulation', + 'description': 'Candor: The Art of Gesticulation', + 'thumbnail': 're:^https?://.*\.jpg', + 'uploader': 'Nowness', + } + }, { + 'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr', + 'md5': 'e79cf125e387216f86b2e0a5b5c63aa3', + 'info_dict': { + 'id': '3716354522001', + 'ext': 'mp4', + 'title': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR', + 'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR', + 'thumbnail': 're:^https?://.*\.jpg', + 'uploader': 'Nowness', + } + }] def _real_extract(self, url): - display_id, post = self.api_request(url, 'post/getBySlug/%s') - return self.extract_url_result(post) + _, post = self._api_request(url, 'post/getBySlug/%s') + return self._extract_url_result(post) class NownessPlaylistIE(NownessBaseIE): @@ -83,33 +82,39 @@ class NownessPlaylistIE(NownessBaseIE): _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/playlist/(?P\d+)' _TEST = { 'url': 'https://www.nowness.com/playlist/3286/i-guess-thats-why-they-call-it-the-blues', - 'info_dict': - { + 'info_dict': { 'id': '3286', }, 'playlist_mincount': 8, } def _real_extract(self, url): - playlist_id, playlist = self.api_request(url, 'post?PlaylistId=%s') - entries = [self.extract_url_result(item) for item in playlist['items']] + playlist_id, playlist = self._api_request(url, 'post?PlaylistId=%s') + entries = [self._extract_url_result(item) for item in playlist['items']] return self.playlist_result(entries, playlist_id) -class NownessSerieIE(NownessBaseIE): - IE_NAME = 'nowness:serie' +class NownessSeriesIE(NownessBaseIE): + IE_NAME = 'nowness:series' _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/series/(?P[^/]+?)(?:$|[?#])' _TEST = { 'url': 'https://www.nowness.com/series/60-seconds', - 'info_dict': - { + 'info_dict': { 'id': '60', + 'title': '60 Seconds', + 'description': 'One-minute wisdom in a new NOWNESS series', }, 'playlist_mincount': 4, } def _real_extract(self, url): - display_id, serie = self.api_request(url, 'series/getBySlug/%s') - serie_id = str(serie['id']) - entries = [self.extract_url_result(post) for post in serie['posts']] - return self.playlist_result(entries, serie_id) + display_id, series = self._api_request(url, 'series/getBySlug/%s') + entries = [self._extract_url_result(post) for post in series['posts']] + series_title = None + series_description = None + translations = series.get('translations', []) + if translations: + series_title = translations[0].get('title') or translations[0]['seoTitle'] + series_description = translations[0].get('seoDescription') + return self.playlist_result( + entries, compat_str(series['id']), series_title, series_description) From c246773599f6b541033ac926ac6ed360fb63f6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 13 Sep 2015 19:27:22 +0600 Subject: [PATCH 1340/1424] [nowness] Add vimeo test --- youtube_dl/extractor/nowness.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py index 9fb916c18..b97f62fdb 100644 --- a/youtube_dl/extractor/nowness.py +++ b/youtube_dl/extractor/nowness.py @@ -58,7 +58,7 @@ class NownessIE(NownessBaseIE): 'description': 'Candor: The Art of Gesticulation', 'thumbnail': 're:^https?://.*\.jpg', 'uploader': 'Nowness', - } + }, }, { 'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr', 'md5': 'e79cf125e387216f86b2e0a5b5c63aa3', @@ -69,7 +69,21 @@ class NownessIE(NownessBaseIE): 'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR', 'thumbnail': 're:^https?://.*\.jpg', 'uploader': 'Nowness', - } + }, + }, { + # vimeo + 'url': 'https://www.nowness.com/series/nowness-picks/jean-luc-godard-supercut', + 'md5': '9a5a6a8edf806407e411296ab6bc2a49', + 'info_dict': { + 'id': '130020913', + 'ext': 'mp4', + 'title': 'Bleu, Blanc, Rouge - A Godard Supercut', + 'description': 'md5:f0ea5f1857dffca02dbd37875d742cec', + 'thumbnail': 're:^https?://.*\.jpg', + 'upload_date': '20150607', + 'uploader': 'Cinema Sem Lei', + 'uploader_id': 'cinemasemlei', + }, }] def _real_extract(self, url): From ac7a1b0dfb9e1e3ea384dd36a425ed9483ee17ef Mon Sep 17 00:00:00 2001 From: "Sergey M." Date: Sun, 13 Sep 2015 19:31:20 +0600 Subject: [PATCH 1341/1424] [README.md] Clarify playlist_index description --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 24bfe38a2..27b20df04 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,7 @@ The `-o` option allows users to indicate a template for the output file names. T - `epoch`: The sequence will be replaced by the Unix epoch when creating the file. - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero. - `playlist`: The name or the id of the playlist that contains the video. - - `playlist_index`: The index of the video in the playlist, a five-digit number. + - `playlist_index`: The sequence will be replaced by the index of the video in the playlist padded with leading zeros according to the total length of the playlist. - `format_id`: The sequence will be replaced by the format code specified by `--format`. The current default template is `%(title)s-%(id)s.%(ext)s`. From 31208a07c247f5b8f1f0fc26da1d477f7440e513 Mon Sep 17 00:00:00 2001 From: "Sergey M." Date: Sun, 13 Sep 2015 19:34:09 +0600 Subject: [PATCH 1342/1424] [README.md] Unify playlist description --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 27b20df04..1a00a02bf 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ The `-o` option allows users to indicate a template for the output file names. T - `ext`: The sequence will be replaced by the appropriate extension (like flv or mp4). - `epoch`: The sequence will be replaced by the Unix epoch when creating the file. - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero. - - `playlist`: The name or the id of the playlist that contains the video. + - `playlist`: The sequence will be replaced by the name or the id of the playlist that contains the video. - `playlist_index`: The sequence will be replaced by the index of the video in the playlist padded with leading zeros according to the total length of the playlist. - `format_id`: The sequence will be replaced by the format code specified by `--format`. From e2ff3df314c7e710d429ca311b2a22839d4aa3f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 14 Sep 2015 03:35:15 +0600 Subject: [PATCH 1343/1424] [clubic] Relax _VALID_URL (Closes #6854) --- youtube_dl/extractor/clubic.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/clubic.py b/youtube_dl/extractor/clubic.py index 14f215c5c..1dfa7c12e 100644 --- a/youtube_dl/extractor/clubic.py +++ b/youtube_dl/extractor/clubic.py @@ -12,9 +12,9 @@ from ..utils import ( class ClubicIE(InfoExtractor): - _VALID_URL = r'http://(?:www\.)?clubic\.com/video/[^/]+/video.*-(?P[0-9]+)\.html' + _VALID_URL = r'http://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P[0-9]+)\.html' - _TEST = { + _TESTS = [{ 'url': 'http://www.clubic.com/video/clubic-week/video-clubic-week-2-0-le-fbi-se-lance-dans-la-photo-d-identite-448474.html', 'md5': '1592b694ba586036efac1776b0b43cd3', 'info_dict': { @@ -24,7 +24,10 @@ class ClubicIE(InfoExtractor): 'description': 're:Gueule de bois chez Nokia. Le constructeur a indiqué cette.*', 'thumbnail': 're:^http://img\.clubic\.com/.*\.jpg$', } - } + }, { + 'url': 'http://www.clubic.com/video/video-clubic-week-2-0-apple-iphone-6s-et-plus-mais-surtout-le-pencil-469792.html', + 'only_matching': True, + }] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) From 953fed280f1a35bc6e8a701a25b8450ec08efdd3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 14 Sep 2015 00:25:08 +0200 Subject: [PATCH 1344/1424] [compat] Do not use unicode If the code ever runs on 3.x, it would fail. Even if it never does, the unicode name confuses Python 3 code analysis tools. --- youtube_dl/compat.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index e32bef279..1ff42d94b 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -80,6 +80,11 @@ try: except ImportError: import BaseHTTPServer as compat_http_server +try: + compat_str = unicode # Python 2 +except NameError: + compat_str = str + try: from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes from urllib.parse import unquote as compat_urllib_parse_unquote @@ -100,7 +105,7 @@ except ImportError: # Python 2 # Is it a string-like object? string.split return b'' - if isinstance(string, unicode): + if isinstance(string, compat_str): string = string.encode('utf-8') bits = string.split(b'%') if len(bits) == 1: @@ -150,11 +155,6 @@ except ImportError: # Python 2 string = string.replace('+', ' ') return compat_urllib_parse_unquote(string, encoding, errors) -try: - compat_str = unicode # Python 2 -except NameError: - compat_str = str - try: compat_basestring = basestring # Python 2 except NameError: @@ -234,7 +234,7 @@ else: # Working around shlex issue with unicode strings on some python 2 # versions (see http://bugs.python.org/issue1548891) def compat_shlex_split(s, comments=False, posix=True): - if isinstance(s, unicode): + if isinstance(s, compat_str): s = s.encode('utf-8') return shlex.split(s, comments, posix) From 287be8c615574f90f4fac5adad0b32b3102f621e Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 14 Sep 2015 00:26:12 +0200 Subject: [PATCH 1345/1424] [youtube:truncated_url] Add ?t= Sometimes found in links - ?t=123&v=ABCD starts the video at position 123. --- youtube_dl/extractor/youtube.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 97ce36550..abacdfaed 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1970,6 +1970,7 @@ class YoutubeTruncatedURLIE(InfoExtractor): annotation_id=annotation_[^&]+| x-yt-cl=[0-9]+| hl=[^&]*| + t=[0-9]+ )? | attribution_link\?a=[^&]+ @@ -1992,6 +1993,9 @@ class YoutubeTruncatedURLIE(InfoExtractor): }, { 'url': 'https://www.youtube.com/watch?hl=en-GB', 'only_matching': True, + }, { + 'url': 'https://www.youtube.com/watch?t=2372', + 'only_matching': True, }] def _real_extract(self, url): From 2b3c2546782f84e0b3fe42b55d2030e5350cbe92 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 14 Sep 2015 00:32:20 +0200 Subject: [PATCH 1346/1424] [youtube:channel] Correct 404 handling Previously, when we encountered a 404 - such as youtube-dl https://www.youtube.com/ohJeiboh8oorehai - we crashed with a regexp error. Instead, make sure to go on and eventually report a 404. --- youtube_dl/extractor/youtube.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index abacdfaed..b252e36e1 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1654,12 +1654,15 @@ class YoutubeChannelIE(InfoExtractor): channel_page = self._download_webpage( url + '?view=57', channel_id, 'Downloading channel page', fatal=False) - channel_playlist_id = self._html_search_meta( - 'channelId', channel_page, 'channel id', default=None) - if not channel_playlist_id: - channel_playlist_id = self._search_regex( - r'data-channel-external-id="([^"]+)"', - channel_page, 'channel id', default=None) + if channel_page is False: + channel_playlist_id = False + else: + channel_playlist_id = self._html_search_meta( + 'channelId', channel_page, 'channel id', default=None) + if not channel_playlist_id: + channel_playlist_id = self._search_regex( + r'data-channel-external-id="([^"]+)"', + channel_page, 'channel id', default=None) if channel_playlist_id and channel_playlist_id.startswith('UC'): playlist_id = 'UU' + channel_playlist_id[2:] return self.url_result( From 3b9264a04902c4ce01d3fac67bf51799fece0e53 Mon Sep 17 00:00:00 2001 From: "Sergey M." Date: Mon, 14 Sep 2015 23:07:03 +0600 Subject: [PATCH 1347/1424] [README.md] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a00a02bf..42656aae1 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,7 @@ For example: machine youtube login myaccount@gmail.com password my_youtube_password machine twitch login my_twitch_account_name password my_twitch_password ``` -To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or to place it in [configuration file](#configuration). +To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or place it in [configuration file](#configuration). On Windows you may also need to setup `%HOME%` environment variable manually. From 0391bc8176c22e586f1c866145bc31e9cd2c50ed Mon Sep 17 00:00:00 2001 From: Alex Vong Date: Mon, 10 Aug 2015 12:55:29 +0800 Subject: [PATCH 1348/1424] Add the missing tag `EMBEDDING YOUTUBE-DL` in the content table. * README.md: Add missing tag. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 42656aae1..2ed751791 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ youtube-dl - download videos from youtube.com or other video platforms - [VIDEO SELECTION](#video-selection) - [FAQ](#faq) - [DEVELOPER INSTRUCTIONS](#developer-instructions) +- [EMBEDDING YOUTUBE-DL](#embedding-youtube-dl) - [BUGS](#bugs) - [COPYRIGHT](#copyright) From 272e4db5c746ef4c94b7c9e7bac0fc008319b158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 15 Sep 2015 21:24:01 +0600 Subject: [PATCH 1349/1424] [pornhub] Relax _VALID_URL (Closes #6868) --- youtube_dl/extractor/pornhub.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py index 7b0cdc41a..a656ad85a 100644 --- a/youtube_dl/extractor/pornhub.py +++ b/youtube_dl/extractor/pornhub.py @@ -20,7 +20,7 @@ from ..aes import ( class PornHubIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P[0-9a-z]+)' + _VALID_URL = r'https?://(?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P[0-9a-z]+)' _TESTS = [{ 'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015', 'md5': '882f488fa1f0026f023f33576004a2ed', @@ -34,6 +34,9 @@ class PornHubIE(InfoExtractor): }, { 'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d', 'only_matching': True, + }, { + 'url': 'http://fr.pornhub.com/view_video.php?viewkey=ph55ca2f9760862', + 'only_matching': True, }] @classmethod From 88060cce108f7436f8747186855ef0d63fa23338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 15 Sep 2015 21:57:30 +0600 Subject: [PATCH 1350/1424] [rai] Add support for videos embedded with drawMediaRaiTV (Closes #6866) --- youtube_dl/extractor/rai.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py index 1631faf29..7ff1d06c4 100644 --- a/youtube_dl/extractor/rai.py +++ b/youtube_dl/extractor/rai.py @@ -5,6 +5,7 @@ import re from .common import InfoExtractor from ..compat import ( compat_urllib_parse, + compat_urlparse, ) from ..utils import ( parse_duration, @@ -72,6 +73,18 @@ class RaiIE(InfoExtractor): 'description': 'Primo appuntamento con "Il candidato" con Filippo Timi, alias Piero Zucca presidente!', 'uploader': 'RaiTre', } + }, + { + 'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html', + 'md5': '037104d2c14132887e5e4cf114569214', + 'info_dict': { + 'id': '0c7a664b-d0f4-4b2c-8835-3f82e46f433e', + 'ext': 'flv', + 'title': 'Il pacco', + 'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a', + 'uploader': 'RaiTre', + 'upload_date': '20141221', + }, } ] @@ -90,11 +103,14 @@ class RaiIE(InfoExtractor): relinker_url = self._extract_relinker_url(webpage) if not relinker_url: - iframe_path = self._search_regex( - r']+src="/?(dl/[^"]+\?iframe\b[^"]*)"', + iframe_url = self._search_regex( + [r']+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"', + r'drawMediaRaiTV\(["\'](.+?)["\']'], webpage, 'iframe') + if not iframe_url.startswith('http'): + iframe_url = compat_urlparse.urljoin(url, iframe_url) webpage = self._download_webpage( - '%s/%s' % (host, iframe_path), video_id) + iframe_url, video_id) relinker_url = self._extract_relinker_url(webpage) relinker = self._download_json( From 12bc2429444b9f89d1b7f4f631c25cc3af2a1d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 16 Sep 2015 22:25:25 +0600 Subject: [PATCH 1351/1424] [telecinco] Incorporate mitele code --- youtube_dl/extractor/telecinco.py | 54 +++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py index ae94f055c..c0ad286c5 100644 --- a/youtube_dl/extractor/telecinco.py +++ b/youtube_dl/extractor/telecinco.py @@ -1,10 +1,22 @@ # coding: utf-8 from __future__ import unicode_literals -from .mitele import MiTeleIE +import json + +from .common import InfoExtractor +from ..compat import ( + compat_urllib_parse, + compat_urllib_parse_unquote, + compat_urlparse, +) +from ..utils import ( + get_element_by_attribute, + parse_duration, + strip_jsonp, +) -class TelecincoIE(MiTeleIE): +class TelecincoIE(InfoExtractor): IE_NAME = 'telecinco.es' _VALID_URL = r'https?://www\.telecinco\.es/(?:[^/]+/)+(?P.+?)\.html' @@ -27,3 +39,41 @@ class TelecincoIE(MiTeleIE): 'url': 'http://www.telecinco.es/espanasinirmaslejos/Espana-gran-destino-turistico_2_1240605043.html', 'only_matching': True, }] + + def _real_extract(self, url): + episode = self._match_id(url) + webpage = self._download_webpage(url, episode) + embed_data_json = self._search_regex( + r'(?s)MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data', + ).replace('\'', '"') + embed_data = json.loads(embed_data_json) + + domain = embed_data['mediaUrl'] + if not domain.startswith('http'): + # only happens in telecinco.es videos + domain = 'http://' + domain + info_url = compat_urlparse.urljoin( + domain, + compat_urllib_parse_unquote(embed_data['flashvars']['host']) + ) + info_el = self._download_xml(info_url, episode).find('./video/info') + + video_link = info_el.find('videoUrl/link').text + token_query = compat_urllib_parse.urlencode({'id': video_link}) + token_info = self._download_json( + embed_data['flashvars']['ov_tk'] + '?' + token_query, + episode, + transform_source=strip_jsonp + ) + formats = self._extract_m3u8_formats( + token_info['tokenizedUrl'], episode, ext='mp4') + + return { + 'id': embed_data['videoId'], + 'display_id': episode, + 'title': info_el.find('title').text, + 'formats': formats, + 'description': get_element_by_attribute('class', 'text', webpage), + 'thumbnail': info_el.find('thumb').text, + 'duration': parse_duration(info_el.find('duration').text), + } From f84ce1ebaf930f5fc422e4fab77fa9c7c86a999a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 16 Sep 2015 22:27:03 +0600 Subject: [PATCH 1352/1424] [mitele] Fix extraction (Closes #6414) --- youtube_dl/extractor/mitele.py | 105 ++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 47 deletions(-) diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py index 852d72266..9e8a8fc72 100644 --- a/youtube_dl/extractor/mitele.py +++ b/youtube_dl/extractor/mitele.py @@ -1,17 +1,11 @@ from __future__ import unicode_literals -import json - from .common import InfoExtractor -from ..compat import ( - compat_urllib_parse, - compat_urllib_parse_unquote, - compat_urlparse, -) +from ..compat import compat_urllib_parse from ..utils import ( + encode_dict, get_element_by_attribute, - parse_duration, - strip_jsonp, + int_or_none, ) @@ -21,54 +15,71 @@ class MiTeleIE(InfoExtractor): _TESTS = [{ 'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/', + 'md5': 'ace7635b2a0b286aaa37d3ff192d2a8a', 'info_dict': { - 'id': '0fce117d', - 'ext': 'mp4', - 'title': 'Programa 144 - Tor, la web invisible', - 'description': 'md5:3b6fce7eaa41b2d97358726378d9369f', + 'id': '0NF1jJnxS1Wu3pHrmvFyw2', 'display_id': 'programa-144', + 'ext': 'flv', + 'title': 'Tor, la web invisible', + 'description': 'md5:3b6fce7eaa41b2d97358726378d9369f', + 'thumbnail': 're:(?i)^https?://.*\.jpg$', 'duration': 2913, }, - 'params': { - # m3u8 download - 'skip_download': True, - }, }] def _real_extract(self, url): - episode = self._match_id(url) - webpage = self._download_webpage(url, episode) - embed_data_json = self._search_regex( - r'(?s)MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data', - ).replace('\'', '"') - embed_data = json.loads(embed_data_json) + display_id = self._match_id(url) - domain = embed_data['mediaUrl'] - if not domain.startswith('http'): - # only happens in telecinco.es videos - domain = 'http://' + domain - info_url = compat_urlparse.urljoin( - domain, - compat_urllib_parse_unquote(embed_data['flashvars']['host']) - ) - info_el = self._download_xml(info_url, episode).find('./video/info') + webpage = self._download_webpage(url, display_id) - video_link = info_el.find('videoUrl/link').text - token_query = compat_urllib_parse.urlencode({'id': video_link}) - token_info = self._download_json( - embed_data['flashvars']['ov_tk'] + '?' + token_query, - episode, - transform_source=strip_jsonp - ) - formats = self._extract_m3u8_formats( - token_info['tokenizedUrl'], episode, ext='mp4') + config_url = self._search_regex( + r'data-config\s*=\s*"([^"]+)"', webpage, 'data config url') + + config = self._download_json( + config_url, display_id, 'Downloading config JSON') + + mmc = self._download_json( + config['services']['mmc'], display_id, 'Downloading mmc JSON') + + formats = [] + for location in mmc['locations']: + gat = self._proto_relative_url(location.get('gat'), 'http:') + bas = location.get('bas') + loc = location.get('loc') + ogn = location.get('ogn') + if None in (gat, bas, loc, ogn): + continue + token_data = { + 'bas': bas, + 'icd': loc, + 'ogn': ogn, + 'sta': '0', + } + media = self._download_json( + '%s/?%s' % (gat, compat_urllib_parse.urlencode(encode_dict(token_data)).encode('utf-8')), + display_id, 'Downloading %s JSON' % location['loc']) + file_ = media.get('file') + if not file_: + continue + formats.extend(self._extract_f4m_formats( + file_ + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18', + display_id, f4m_id=loc)) + + title = self._search_regex( + r'class="Destacado-text"[^>]*>\s*([^<]+)', webpage, 'title') + + video_id = self._search_regex( + r'data-media-id\s*=\s*"([^"]+)"', webpage, + 'data media id', default=None) or display_id + thumbnail = config.get('poster', {}).get('imageUrl') + duration = int_or_none(mmc.get('duration')) return { - 'id': embed_data['videoId'], - 'display_id': episode, - 'title': info_el.find('title').text, - 'formats': formats, + 'id': video_id, + 'display_id': display_id, + 'title': title, 'description': get_element_by_attribute('class', 'text', webpage), - 'thumbnail': info_el.find('thumb').text, - 'duration': parse_duration(info_el.find('duration').text), + 'thumbnail': thumbnail, + 'duration': duration, + 'formats': formats, } From d5e7657fe2c045545e3cdc089413661befad8b4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 16 Sep 2015 22:37:01 +0600 Subject: [PATCH 1353/1424] [telecinco] Add support or cuatro.com --- youtube_dl/extractor/telecinco.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py index c0ad286c5..018fd7d34 100644 --- a/youtube_dl/extractor/telecinco.py +++ b/youtube_dl/extractor/telecinco.py @@ -18,7 +18,7 @@ from ..utils import ( class TelecincoIE(InfoExtractor): IE_NAME = 'telecinco.es' - _VALID_URL = r'https?://www\.telecinco\.es/(?:[^/]+/)+(?P.+?)\.html' + _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com)/(?:[^/]+/)+(?P.+?)\.html' _TESTS = [{ 'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html', @@ -32,6 +32,15 @@ class TelecincoIE(InfoExtractor): # m3u8 download 'skip_download': True, }, + }, { + 'url': 'http://www.cuatro.com/deportes/futbol/barcelona/Leo_Messi-Champions-Roma_2_2052780128.html', + 'md5': '0a5b9f3cc8b074f50a0578f823a12694', + 'info_dict': { + 'id': 'MDSVID20150916_0128', + 'ext': 'mp4', + 'title': '¿Quién es este ex futbolista con el que hablan ...', + 'duration': 79, + }, }, { 'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html', 'only_matching': True, @@ -66,7 +75,7 @@ class TelecincoIE(InfoExtractor): transform_source=strip_jsonp ) formats = self._extract_m3u8_formats( - token_info['tokenizedUrl'], episode, ext='mp4') + token_info['tokenizedUrl'], episode, ext='mp4', entry_protocol='m3u8_native') return { 'id': embed_data['videoId'], From 369e60162e9cc99c89dac52a40297b68cbc82392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 16 Sep 2015 22:37:47 +0600 Subject: [PATCH 1354/1424] [telecinco] Update test --- youtube_dl/extractor/telecinco.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py index 018fd7d34..685f1db40 100644 --- a/youtube_dl/extractor/telecinco.py +++ b/youtube_dl/extractor/telecinco.py @@ -22,16 +22,13 @@ class TelecincoIE(InfoExtractor): _TESTS = [{ 'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html', + 'md5': '5cbef3ad5ef17bf0d21570332d140729', 'info_dict': { 'id': 'MDSVID20141015_0058', 'ext': 'mp4', 'title': 'Con Martín Berasategui, hacer un bacalao al ...', 'duration': 662, }, - 'params': { - # m3u8 download - 'skip_download': True, - }, }, { 'url': 'http://www.cuatro.com/deportes/futbol/barcelona/Leo_Messi-Champions-Roma_2_2052780128.html', 'md5': '0a5b9f3cc8b074f50a0578f823a12694', From 0e1b2566fff795981d0f7cccbca6700a3fc260a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 16 Sep 2015 22:39:24 +0600 Subject: [PATCH 1355/1424] [telecinco] Clarify IE_DESC --- youtube_dl/extractor/telecinco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py index 685f1db40..9740c3452 100644 --- a/youtube_dl/extractor/telecinco.py +++ b/youtube_dl/extractor/telecinco.py @@ -17,7 +17,7 @@ from ..utils import ( class TelecincoIE(InfoExtractor): - IE_NAME = 'telecinco.es' + IE_DESC = 'telecinco.es and cuatro.es' _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com)/(?:[^/]+/)+(?P.+?)\.html' _TESTS = [{ From 3368d70dceba93aa9acdb97ec2705471a3424c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 16 Sep 2015 22:39:48 +0600 Subject: [PATCH 1356/1424] [mitele] Clarify IE_DESC --- youtube_dl/extractor/mitele.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py index 9e8a8fc72..54993e2c9 100644 --- a/youtube_dl/extractor/mitele.py +++ b/youtube_dl/extractor/mitele.py @@ -10,7 +10,7 @@ from ..utils import ( class MiTeleIE(InfoExtractor): - IE_NAME = 'mitele.es' + IE_DESC = 'mitele.es' _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P[^/]+)/' _TESTS = [{ From d492dad8f440c8d0d006a896de0a56b01cd46496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 16 Sep 2015 22:45:39 +0600 Subject: [PATCH 1357/1424] [telecinco] Add support for mediaset.es --- youtube_dl/extractor/telecinco.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py index 9740c3452..b0bf9ab8d 100644 --- a/youtube_dl/extractor/telecinco.py +++ b/youtube_dl/extractor/telecinco.py @@ -17,8 +17,8 @@ from ..utils import ( class TelecincoIE(InfoExtractor): - IE_DESC = 'telecinco.es and cuatro.es' - _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com)/(?:[^/]+/)+(?P.+?)\.html' + IE_DESC = 'telecinco.es, cuatro.es and mediaset.es' + _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com|mediaset\.es)/(?:[^/]+/)+(?P.+?)\.html' _TESTS = [{ 'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html', @@ -38,6 +38,15 @@ class TelecincoIE(InfoExtractor): 'title': '¿Quién es este ex futbolista con el que hablan ...', 'duration': 79, }, + }, { + 'url': 'http://www.mediaset.es/12meses/campanas/doylacara/conlatratanohaytrato/Ayudame-dar-cara-trata-trato_2_1986630220.html', + 'md5': 'ad1bfaaba922dd4a295724b05b68f86a', + 'info_dict': { + 'id': 'MDSVID20150513_0220', + 'ext': 'mp4', + 'title': '#DOYLACARA. Con la trata no hay trato', + 'duration': 50, + }, }, { 'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html', 'only_matching': True, From 4647fd89103dc5ded16dc069828a2cf38216093c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 16 Sep 2015 22:49:26 +0600 Subject: [PATCH 1358/1424] [telecinco] Fix typo --- youtube_dl/extractor/telecinco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py index b0bf9ab8d..2c8e9b941 100644 --- a/youtube_dl/extractor/telecinco.py +++ b/youtube_dl/extractor/telecinco.py @@ -17,7 +17,7 @@ from ..utils import ( class TelecincoIE(InfoExtractor): - IE_DESC = 'telecinco.es, cuatro.es and mediaset.es' + IE_DESC = 'telecinco.es, cuatro.com and mediaset.es' _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com|mediaset\.es)/(?:[^/]+/)+(?P.+?)\.html' _TESTS = [{ From 659ffe204c946b977266aa373110fe50a2763066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 17 Sep 2015 22:58:49 +0600 Subject: [PATCH 1359/1424] [divxstage] Remove extractor DivxStage is offline for quite some time already --- youtube_dl/extractor/__init__.py | 1 - youtube_dl/extractor/divxstage.py | 27 --------------------------- 2 files changed, 28 deletions(-) delete mode 100644 youtube_dl/extractor/divxstage.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 74b7df463..c16e9e3d2 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -138,7 +138,6 @@ from .dump import DumpIE from .dumpert import DumpertIE from .defense import DefenseGouvFrIE from .discovery import DiscoveryIE -from .divxstage import DivxStageIE from .dropbox import DropboxIE from .eagleplatform import EaglePlatformIE from .ebaumsworld import EbaumsWorldIE diff --git a/youtube_dl/extractor/divxstage.py b/youtube_dl/extractor/divxstage.py deleted file mode 100644 index b88379e06..000000000 --- a/youtube_dl/extractor/divxstage.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import unicode_literals - -from .novamov import NovaMovIE - - -class DivxStageIE(NovaMovIE): - IE_NAME = 'divxstage' - IE_DESC = 'DivxStage' - - _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'divxstage\.(?:eu|net|ch|co|at|ag|to)'} - - _HOST = 'www.divxstage.eu' - - _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<' - _TITLE_REGEX = r'
\s*([^<]+)' - _DESCRIPTION_REGEX = r'
\s*[^<]+\s*

([^<]+)

' - - _TEST = { - 'url': 'http://www.divxstage.eu/video/57f238e2e5e01', - 'md5': '63969f6eb26533a1968c4d325be63e72', - 'info_dict': { - 'id': '57f238e2e5e01', - 'ext': 'flv', - 'title': 'youtubedl test video', - 'description': 'This is a test video for youtubedl.', - } - } From 1806a75415a3b62b07ab3fd282f915c6af14dfd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 17 Sep 2015 22:59:15 +0600 Subject: [PATCH 1360/1424] [ok] Capture error message --- youtube_dl/extractor/odnoklassniki.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py index 66520c2c5..0cd96c10a 100644 --- a/youtube_dl/extractor/odnoklassniki.py +++ b/youtube_dl/extractor/odnoklassniki.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..compat import compat_urllib_parse_unquote from ..utils import ( + ExtractorError, unified_strdate, int_or_none, qualities, @@ -72,6 +73,12 @@ class OdnoklassnikiIE(InfoExtractor): webpage = self._download_webpage( 'http://ok.ru/video/%s' % video_id, video_id) + error = self._search_regex( + r'[^>]+class="vp_video_stub_txt"[^>]*>([^<]+)<', + webpage, 'error', default=None) + if error: + raise ExtractorError(error, expected=True) + player = self._parse_json( unescapeHTML(self._search_regex( r'data-options=(?P["\'])(?P{.+?%s.+?})(?P=quote)' % video_id, From 131d05033bfe3f5a8ef3241489a44dc5c1e03830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 17 Sep 2015 22:59:32 +0600 Subject: [PATCH 1361/1424] [ok] Skip test --- youtube_dl/extractor/odnoklassniki.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py index 0cd96c10a..ccc88cfb1 100644 --- a/youtube_dl/extractor/odnoklassniki.py +++ b/youtube_dl/extractor/odnoklassniki.py @@ -29,6 +29,7 @@ class OdnoklassnikiIE(InfoExtractor): 'like_count': int, 'age_limit': 0, }, + 'skip': 'Video has been blocked', }, { # metadataUrl 'url': 'http://ok.ru/video/63567059965189-0', From 8a7bbd160641dbbc13f68aaa26dfa49b71483a8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 17 Sep 2015 15:22:19 +0200 Subject: [PATCH 1362/1424] [postprocessor/ffmpeg] Always use the 'file:' protocol for filenames (fixes #6874) If the filename contains ':' it is interpreted as a protocol. It also handles filenames starting with '-'. --- youtube_dl/downloader/hls.py | 2 +- youtube_dl/postprocessor/ffmpeg.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py index b2436e732..7743e176a 100644 --- a/youtube_dl/downloader/hls.py +++ b/youtube_dl/downloader/hls.py @@ -31,7 +31,7 @@ class HlsFD(FileDownloader): args = [ encodeArgument(opt) for opt in (ffpp.executable, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')] - args.append(encodeFilename(tmpfilename, True)) + args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True)) self._debug_cmd(args) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 1f723908b..4f320e124 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -135,7 +135,10 @@ class FFmpegPostProcessor(PostProcessor): files_cmd = [] for path in input_paths: - files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)]) + files_cmd.extend([ + encodeArgument('-i'), + encodeFilename(self._ffmpeg_filename_argument(path), True) + ]) cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] + files_cmd + [encodeArgument(o) for o in opts] + @@ -155,10 +158,10 @@ class FFmpegPostProcessor(PostProcessor): self.run_ffmpeg_multiple_files([path], out_path, opts) def _ffmpeg_filename_argument(self, fn): - # ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details - if fn.startswith('-'): - return './' + fn - return fn + # Always use 'file:' because the filename may contain ':' (ffmpeg + # interprets that as a protocol) or can start with '-' (-- is broken in + # ffmpeg, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details) + return 'file:' + fn class FFmpegExtractAudioPP(FFmpegPostProcessor): From 06368a232a9bc8b383db77b0087a2ab1af94a833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 18 Sep 2015 21:01:55 +0600 Subject: [PATCH 1363/1424] [playwire] Fix test --- youtube_dl/extractor/playwire.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/playwire.py b/youtube_dl/extractor/playwire.py index bdc71017b..6d138ef25 100644 --- a/youtube_dl/extractor/playwire.py +++ b/youtube_dl/extractor/playwire.py @@ -19,7 +19,7 @@ class PlaywireIE(InfoExtractor): 'id': '3353705', 'ext': 'mp4', 'title': 'S04_RM_UCL_Rus', - 'thumbnail': 're:^http://.*\.png$', + 'thumbnail': 're:^https?://.*\.png$', 'duration': 145.94, }, }, { From d1c694ea4ad75eb14e02e27ba6db5be95a598f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 19 Sep 2015 00:51:41 +0600 Subject: [PATCH 1364/1424] [bbc] Switch extraction to hq iptv hls (Closes #2732, closes #3023, closes #4828, closes #6803, closes #6855) --- youtube_dl/extractor/bbc.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py index abc5a44a1..42526357a 100644 --- a/youtube_dl/extractor/bbc.py +++ b/youtube_dl/extractor/bbc.py @@ -21,6 +21,7 @@ class BBCCoUkIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P[\da-z]{8})' _MEDIASELECTOR_URLS = [ + 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s', 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s', ] @@ -189,6 +190,12 @@ class BBCCoUkIE(InfoExtractor): # Skip DASH until supported elif transfer_format == 'dash': pass + elif transfer_format == 'hls': + m3u8_formats = self._extract_m3u8_formats( + href, programme_id, ext='mp4', entry_protocol='m3u8_native', + m3u8_id=supplier, fatal=False) + if m3u8_formats: + formats.extend(m3u8_formats) # Direct link else: formats.append({ From 1d67c9664099c261b7752cb56773e423d410fae6 Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 18 Sep 2015 22:09:33 +0100 Subject: [PATCH 1365/1424] [shahid] fix api request url --- youtube_dl/extractor/shahid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py index 6e9903d5e..3fd545373 100644 --- a/youtube_dl/extractor/shahid.py +++ b/youtube_dl/extractor/shahid.py @@ -81,7 +81,7 @@ class ShahidIE(InfoExtractor): compat_urllib_parse.urlencode({ 'apiKey': 'sh@hid0nlin3', 'hash': 'b2wMCTHpSmyxGqQjJFOycRmLSex+BpTK/ooxy6vHaqs=', - }).encode('utf-8')), + })), video_id, 'Downloading video JSON') video = video[api_vars['playerType']] From 5de5ab89b4136e7f2191f50d8d69587b90e0c3bb Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 18 Sep 2015 22:10:36 +0100 Subject: [PATCH 1366/1424] [shahid] change test ext --- youtube_dl/extractor/shahid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py index 3fd545373..f76fb12c0 100644 --- a/youtube_dl/extractor/shahid.py +++ b/youtube_dl/extractor/shahid.py @@ -16,7 +16,7 @@ class ShahidIE(InfoExtractor): 'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html', 'info_dict': { 'id': '90574', - 'ext': 'm3u8', + 'ext': 'mp4', 'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3', 'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان', 'duration': 2972, From 6c91a5a7f5408cf666f3ee40b53c0d9e42521b88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 20 Sep 2015 11:16:12 +0600 Subject: [PATCH 1367/1424] [extractor/generic] Fix following redirect in Refresh HTTP header on python 2 --- youtube_dl/extractor/generic.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index ec748ed9f..8881a8a23 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import os import re +import sys from .common import InfoExtractor from .youtube import YoutubeIE @@ -230,6 +231,22 @@ class GenericIE(InfoExtractor): 'skip_download': False, } }, + { + # redirect in Refresh HTTP header + 'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1', + 'info_dict': { + 'id': 'pO8h3EaFRdo', + 'ext': 'mp4', + 'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set', + 'description': 'md5:6294cc1af09c4049e0652b51a2df10d5', + 'upload_date': '20150917', + 'uploader_id': 'brtvofficial', + 'uploader': 'Boiler Room', + }, + 'params': { + 'skip_download': False, + }, + }, { 'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html', 'md5': '85b90ccc9d73b4acd9138d3af4c27f89', @@ -1808,6 +1825,9 @@ class GenericIE(InfoExtractor): # Look also in Refresh HTTP header refresh_header = head_response.headers.get('Refresh') if refresh_header: + # In python 2 response HTTP headers are bytestrings + if sys.version_info < (3, 0) and isinstance(refresh_header, str): + refresh_header = refresh_header.decode('iso-8859-1') found = re.search(REDIRECT_REGEX, refresh_header) if found: new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1))) From f817adc4689a2064fcab733d6aebf83fd0e2cff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 20 Sep 2015 11:31:23 +0600 Subject: [PATCH 1368/1424] [youtube:history] Disable extractor until #6893 is investigated Wiped out part of my history as well --- youtube_dl/extractor/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index c16e9e3d2..b93156232 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -808,7 +808,7 @@ from .youtube import ( YoutubeIE, YoutubeChannelIE, YoutubeFavouritesIE, - YoutubeHistoryIE, + #YoutubeHistoryIE, YoutubePlaylistIE, YoutubeRecommendedIE, YoutubeSearchDateIE, From 393ca8c94d1adb1490b23265370ce69043b92546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 20 Sep 2015 11:45:19 +0600 Subject: [PATCH 1369/1424] [arte:+7] Look for json vp url in iframe (Closes #6895) --- youtube_dl/extractor/arte.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 76de24477..9ecb6786c 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -4,6 +4,10 @@ from __future__ import unicode_literals import re from .common import InfoExtractor +from ..compat import ( + compat_parse_qs, + compat_urllib_parse_urlparse, +) from ..utils import ( find_xpath_attr, unified_strdate, @@ -77,7 +81,13 @@ class ArteTVPlus7IE(InfoExtractor): def _extract_from_webpage(self, webpage, video_id, lang): json_url = self._html_search_regex( [r'arte_vp_url=["\'](.*?)["\']', r'data-url=["\']([^"]+)["\']'], - webpage, 'json vp url') + webpage, 'json vp url', default=None) + if not json_url: + iframe_url = self._html_search_regex( + r']+src=(["\'])(?P.+\bjson_url=.+?)\1', + webpage, 'iframe url', group='iframe url') + json_url = compat_parse_qs( + compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0] return self._extract_from_json_url(json_url, video_id, lang) def _extract_from_json_url(self, json_url, video_id, lang): From 5e39123b3b6fd740d133fd45824b49b4bc674fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 20 Sep 2015 11:47:03 +0600 Subject: [PATCH 1370/1424] [arte:+7] Fix typo --- youtube_dl/extractor/arte.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 9ecb6786c..2a00da3ee 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -85,7 +85,7 @@ class ArteTVPlus7IE(InfoExtractor): if not json_url: iframe_url = self._html_search_regex( r']+src=(["\'])(?P.+\bjson_url=.+?)\1', - webpage, 'iframe url', group='iframe url') + webpage, 'iframe url', group='url') json_url = compat_parse_qs( compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0] return self._extract_from_json_url(json_url, video_id, lang) From f005f96ea594a82969885ecd124b094c7e09949a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sun, 20 Sep 2015 12:23:13 +0200 Subject: [PATCH 1371/1424] [youtube:history] Explain why it has disabled and skip test --- test/test_all_urls.py | 2 +- youtube_dl/extractor/__init__.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_all_urls.py b/test/test_all_urls.py index a9db42b30..a929afd15 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -99,7 +99,7 @@ class TestAllURLsMatching(unittest.TestCase): def test_keywords(self): self.assertMatch(':ytsubs', ['youtube:subscriptions']) self.assertMatch(':ytsubscriptions', ['youtube:subscriptions']) - self.assertMatch(':ythistory', ['youtube:history']) + # self.assertMatch(':ythistory', ['youtube:history']) self.assertMatch(':thedailyshow', ['ComedyCentralShows']) self.assertMatch(':tds', ['ComedyCentralShows']) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index b93156232..2529d8657 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -808,6 +808,8 @@ from .youtube import ( YoutubeIE, YoutubeChannelIE, YoutubeFavouritesIE, + # disabled because it can wipe the watch history (see #6893) + # remember to uncumment test in test/test_all_urls when it's fixed #YoutubeHistoryIE, YoutubePlaylistIE, YoutubeRecommendedIE, From 5a1a2e94548f25b5fd540090af3f32a1d875f6b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 20 Sep 2015 21:08:29 +0600 Subject: [PATCH 1372/1424] [utils] Fix kwargs on old python 2 (Closes #6905) --- youtube_dl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 206dd56bc..1dc3153fd 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -619,7 +619,7 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs): # expected HTTP responses to meet HTTP/1.0 or later (see also # https://github.com/rg3/youtube-dl/issues/6727) if sys.version_info < (3, 0): - kwargs['strict'] = True + kwargs[b'strict'] = True hc = http_class(*args, **kwargs) source_address = ydl_handler._params.get('source_address') if source_address is not None: From 5b4c54631a8888d75fa766c0bd6ec1822e6caec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 20 Sep 2015 23:12:40 +0600 Subject: [PATCH 1373/1424] [nfl] Add team domains (#6907) --- youtube_dl/extractor/nfl.py | 132 +++++++++++++++++++++++------------- 1 file changed, 86 insertions(+), 46 deletions(-) diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py index dc54634a5..fe143ef88 100644 --- a/youtube_dl/extractor/nfl.py +++ b/youtube_dl/extractor/nfl.py @@ -16,53 +16,93 @@ from ..utils import ( class NFLIE(InfoExtractor): IE_NAME = 'nfl.com' - _VALID_URL = r'''(?x)https?:// - (?P(?:www\.)?(?:nfl\.com|.*?\.clubs\.nfl\.com))/ - (?:.+?/)* - (?P(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12}))''' - _TESTS = [ - { - 'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights', - 'md5': '394ef771ddcd1354f665b471d78ec4c6', - 'info_dict': { - 'id': '0ap3000000398478', - 'ext': 'mp4', - 'title': 'Week 3: Redskins vs. Eagles highlights', - 'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478', - 'upload_date': '20140921', - 'timestamp': 1411337580, - 'thumbnail': 're:^https?://.*\.jpg$', - } - }, - { - 'url': 'http://prod.www.steelers.clubs.nfl.com/video-and-audio/videos/LIVE_Post_Game_vs_Browns/9d72f26a-9e2b-4718-84d3-09fb4046c266', - 'md5': 'cf85bdb4bc49f6e9d3816d130c78279c', - 'info_dict': { - 'id': '9d72f26a-9e2b-4718-84d3-09fb4046c266', - 'ext': 'mp4', - 'title': 'LIVE: Post Game vs. Browns', - 'description': 'md5:6a97f7e5ebeb4c0e69a418a89e0636e8', - 'upload_date': '20131229', - 'timestamp': 1388354455, - 'thumbnail': 're:^https?://.*\.jpg$', - } - }, - { - 'url': 'http://www.nfl.com/news/story/0ap3000000467586/article/patriots-seahawks-involved-in-lategame-skirmish', - 'info_dict': { - 'id': '0ap3000000467607', - 'ext': 'mp4', - 'title': 'Frustrations flare on the field', - 'description': 'Emotions ran high at the end of the Super Bowl on both sides of the ball after a dramatic finish.', - 'timestamp': 1422850320, - 'upload_date': '20150202', - }, - }, - { - 'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood', - 'only_matching': True, + _VALID_URL = r'''(?x) + https?:// + (?P + (?:www\.)? + (?: + (?: + nfl| + buffalobills| + miamidolphins| + patriots| + newyorkjets| + baltimoreravens| + bengals| + clevelandbrowns| + steelers| + houstontexans| + colts| + jaguars| + titansonline| + denverbroncos| + kcchiefs| + raiders| + chargers| + dallascowboys| + giants| + philadelphiaeagles| + redskins| + chicagobears| + detroitlions| + packers| + vikings| + atlantafalcons| + panthers| + neworleanssaints| + buccaneers| + azcardinals| + stlouisrams| + 49ers| + seahawks + )\.com| + .+?\.clubs\.nfl\.com + ) + )/ + (?:.+?/)* + (?P(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12})) + ''' + _TESTS = [{ + 'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights', + 'md5': '394ef771ddcd1354f665b471d78ec4c6', + 'info_dict': { + 'id': '0ap3000000398478', + 'ext': 'mp4', + 'title': 'Week 3: Redskins vs. Eagles highlights', + 'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478', + 'upload_date': '20140921', + 'timestamp': 1411337580, + 'thumbnail': 're:^https?://.*\.jpg$', } - ] + }, { + 'url': 'http://prod.www.steelers.clubs.nfl.com/video-and-audio/videos/LIVE_Post_Game_vs_Browns/9d72f26a-9e2b-4718-84d3-09fb4046c266', + 'md5': 'cf85bdb4bc49f6e9d3816d130c78279c', + 'info_dict': { + 'id': '9d72f26a-9e2b-4718-84d3-09fb4046c266', + 'ext': 'mp4', + 'title': 'LIVE: Post Game vs. Browns', + 'description': 'md5:6a97f7e5ebeb4c0e69a418a89e0636e8', + 'upload_date': '20131229', + 'timestamp': 1388354455, + 'thumbnail': 're:^https?://.*\.jpg$', + } + }, { + 'url': 'http://www.nfl.com/news/story/0ap3000000467586/article/patriots-seahawks-involved-in-lategame-skirmish', + 'info_dict': { + 'id': '0ap3000000467607', + 'ext': 'mp4', + 'title': 'Frustrations flare on the field', + 'description': 'Emotions ran high at the end of the Super Bowl on both sides of the ball after a dramatic finish.', + 'timestamp': 1422850320, + 'upload_date': '20150202', + }, + }, { + 'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood', + 'only_matching': True, + }, { + 'url': 'http://www.buffalobills.com/video/videos/Rex_Ryan_Show_World_Wide_Rex/b1dcfab2-3190-4bb1-bfc0-d6e603d6601a', + 'only_matching': True, + }] @staticmethod def prepend_host(host, url): From 4423eba49b1ca5e9cfb78370ab0eee8f84fb72bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 20 Sep 2015 23:45:01 +0600 Subject: [PATCH 1374/1424] [nfl] Add support for URLs without id (Closes #6907) --- youtube_dl/extractor/nfl.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py index fe143ef88..55dc6107d 100644 --- a/youtube_dl/extractor/nfl.py +++ b/youtube_dl/extractor/nfl.py @@ -60,7 +60,7 @@ class NFLIE(InfoExtractor): ) )/ (?:.+?/)* - (?P(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12})) + (?P[^/#?&]+) ''' _TESTS = [{ 'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights', @@ -96,6 +96,17 @@ class NFLIE(InfoExtractor): 'timestamp': 1422850320, 'upload_date': '20150202', }, + }, { + 'url': 'http://www.patriots.com/video/2015/09/18/10-days-gillette', + 'md5': '4c319e2f625ffd0b481b4382c6fc124c', + 'info_dict': { + 'id': 'n-238346', + 'ext': 'mp4', + 'title': '10 Days at Gillette', + 'description': 'md5:8cd9cd48fac16de596eadc0b24add951', + 'timestamp': 1442618809, + 'upload_date': '20150918', + }, }, { 'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood', 'only_matching': True, @@ -135,13 +146,14 @@ class NFLIE(InfoExtractor): webpage = self._download_webpage(url, video_id) config_url = NFLIE.prepend_host(host, self._search_regex( - r'(?:config|configURL)\s*:\s*"([^"]+)"', webpage, 'config URL', - default='static/content/static/config/video/config.json')) + r'(?:(?:config|configURL)\s*:\s*|]+data-config\s*=\s*)(["\'])(?P.+?)\1', + webpage, 'config URL', default='static/content/static/config/video/config.json', + group='config')) # For articles, the id in the url is not the video id video_id = self._search_regex( - r'contentId\s*:\s*"([^"]+)"', webpage, 'video id', default=video_id) - config = self._download_json(config_url, video_id, - note='Downloading player config') + r'(?:]+data-contentId\s*=\s*|contentId\s*:\s*)(["\'])(?P.+?)\1', + webpage, 'video id', default=video_id, group='id') + config = self._download_json(config_url, video_id, 'Downloading player config') url_template = NFLIE.prepend_host( host, '{contentURLTemplate:}'.format(**config)) video_data = self._download_json( From 82c06a40acb41df77ee55acf8979eb5f0cfba4c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 20 Sep 2015 23:54:05 +0600 Subject: [PATCH 1375/1424] Fix typo --- youtube_dl/extractor/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 2529d8657..35d7f0dd2 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -809,8 +809,8 @@ from .youtube import ( YoutubeChannelIE, YoutubeFavouritesIE, # disabled because it can wipe the watch history (see #6893) - # remember to uncumment test in test/test_all_urls when it's fixed - #YoutubeHistoryIE, + # remember to uncomment test in test/test_all_urls when it's fixed + YoutubeHistoryIE, YoutubePlaylistIE, YoutubeRecommendedIE, YoutubeSearchDateIE, From 9f5e8d16b3c7a1a2731cbab9661a920fbe04e09f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 21 Sep 2015 01:53:28 +0600 Subject: [PATCH 1376/1424] [youtube:history] Disable exractor --- youtube_dl/extractor/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 35d7f0dd2..030c00790 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -810,7 +810,7 @@ from .youtube import ( YoutubeFavouritesIE, # disabled because it can wipe the watch history (see #6893) # remember to uncomment test in test/test_all_urls when it's fixed - YoutubeHistoryIE, + #YoutubeHistoryIE, YoutubePlaylistIE, YoutubeRecommendedIE, YoutubeSearchDateIE, From c6aa838b51484ad3d036db21ba4e9e834720a693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 21 Sep 2015 21:28:02 +0600 Subject: [PATCH 1377/1424] [youtube:history] Enable exractor --- test/test_all_urls.py | 2 +- youtube_dl/extractor/__init__.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test_all_urls.py b/test/test_all_urls.py index a929afd15..a9db42b30 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -99,7 +99,7 @@ class TestAllURLsMatching(unittest.TestCase): def test_keywords(self): self.assertMatch(':ytsubs', ['youtube:subscriptions']) self.assertMatch(':ytsubscriptions', ['youtube:subscriptions']) - # self.assertMatch(':ythistory', ['youtube:history']) + self.assertMatch(':ythistory', ['youtube:history']) self.assertMatch(':thedailyshow', ['ComedyCentralShows']) self.assertMatch(':tds', ['ComedyCentralShows']) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 030c00790..c16e9e3d2 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -808,9 +808,7 @@ from .youtube import ( YoutubeIE, YoutubeChannelIE, YoutubeFavouritesIE, - # disabled because it can wipe the watch history (see #6893) - # remember to uncomment test in test/test_all_urls when it's fixed - #YoutubeHistoryIE, + YoutubeHistoryIE, YoutubePlaylistIE, YoutubeRecommendedIE, YoutubeSearchDateIE, From 92085e7099d3607ff512f241454d6f6f4535b05a Mon Sep 17 00:00:00 2001 From: remitamine Date: Sun, 20 Sep 2015 22:26:23 +0100 Subject: [PATCH 1378/1424] [viewster] accept https links and fix api_token extraction and extract mp4 video link(fixes #6787) --- youtube_dl/extractor/viewster.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py index cda02ba24..c68a38167 100644 --- a/youtube_dl/extractor/viewster.py +++ b/youtube_dl/extractor/viewster.py @@ -16,7 +16,7 @@ from ..utils import ( class ViewsterIE(InfoExtractor): - _VALID_URL = r'http://(?:www\.)?viewster\.com/(?:serie|movie)/(?P\d+-\d+-\d+)' + _VALID_URL = r'https?://(?:www\.)?viewster\.com/(?:serie|movie)/(?P\d+-\d+-\d+)' _TESTS = [{ # movie, Type=Movie 'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/', @@ -74,8 +74,8 @@ class ViewsterIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) # Get 'api_token' cookie - self._request_webpage(HEADRequest(url), video_id) - cookies = self._get_cookies(url) + self._request_webpage(HEADRequest('http://www.viewster.com/'), video_id) + cookies = self._get_cookies('http://www.viewster.com/') self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value) info = self._download_json( @@ -98,7 +98,7 @@ class ViewsterIE(InfoExtractor): return self.playlist_result(entries, video_id, title, description) formats = [] - for media_type in ('application/f4m+xml', 'application/x-mpegURL'): + for media_type in ('application/f4m+xml', 'application/x-mpegURL', 'video/mp4'): media = self._download_json( 'https://public-api.viewster.com/movies/%s/video?mediaType=%s' % (entry_id, compat_urllib_parse.quote(media_type)), @@ -122,6 +122,8 @@ class ViewsterIE(InfoExtractor): else: formats.append({ 'url': video_url, + 'height': int_or_none(media.get('Height')), + 'width': int_or_none(media.get('Width')), }) self._sort_formats(formats) From 8e97596b7b62fb35ccfeb37c0ce5cb008b217d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 22 Sep 2015 21:47:56 +0600 Subject: [PATCH 1379/1424] [viewster] Extract height from bitrate and prefer mp4 videos --- youtube_dl/extractor/viewster.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py index c68a38167..c6aa73c0e 100644 --- a/youtube_dl/extractor/viewster.py +++ b/youtube_dl/extractor/viewster.py @@ -120,11 +120,18 @@ class ViewsterIE(InfoExtractor): fatal=False # m3u8 sometimes fail )) else: - formats.append({ + format_id = media.get('Bitrate') + f = { 'url': video_url, + 'format_id': 'mp4-%s' % format_id, 'height': int_or_none(media.get('Height')), 'width': int_or_none(media.get('Width')), - }) + 'preference': 1, + } + if format_id and not f['height']: + f['height'] = int_or_none(self._search_regex( + r'^(\d+)[pP]$', format_id, 'height', default=None)) + formats.append(f) self._sort_formats(formats) synopsis = info.get('Synopsis', {}) From cb4e421901ce5fe26b457c33c141267c5c18a237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 22 Sep 2015 21:49:29 +0600 Subject: [PATCH 1380/1424] [voewster] Update tests --- youtube_dl/extractor/viewster.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py index c6aa73c0e..8defc1800 100644 --- a/youtube_dl/extractor/viewster.py +++ b/youtube_dl/extractor/viewster.py @@ -20,10 +20,10 @@ class ViewsterIE(InfoExtractor): _TESTS = [{ # movie, Type=Movie 'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/', - 'md5': '14d3cfffe66d57b41ae2d9c873416f01', + 'md5': 'e642d1b27fcf3a4ffa79f194f5adde36', 'info_dict': { 'id': '1140-11855-000', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'The listening Project', 'description': 'md5:bac720244afd1a8ea279864e67baa071', 'timestamp': 1214870400, @@ -33,10 +33,10 @@ class ViewsterIE(InfoExtractor): }, { # series episode, Type=Episode 'url': 'http://www.viewster.com/serie/1284-19427-001/the-world-and-a-wall/', - 'md5': 'd5434c80fcfdb61651cc2199a88d6ba3', + 'md5': '9243079a8531809efe1b089db102c069', 'info_dict': { 'id': '1284-19427-001', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'The World and a Wall', 'description': 'md5:24814cf74d3453fdf5bfef9716d073e3', 'timestamp': 1428192000, From c430802e32932868e24a6e43d0845df963320829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 22 Sep 2015 21:50:20 +0600 Subject: [PATCH 1381/1424] [extractor/common] Add raise_geo_restricted --- youtube_dl/extractor/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index d694e818e..1e7db8a9b 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -516,6 +516,12 @@ class InfoExtractor(object): '%s. Use --username and --password or --netrc to provide account credentials.' % msg, expected=True) + @staticmethod + def raise_geo_restricted(msg='This video is not available from your location due to geo restriction'): + raise ExtractorError( + '%s. You might want to use --proxy to workaround.' % msg, + expected=True) + # Methods for following #608 @staticmethod def url_result(url, ie=None, video_id=None, video_title=None): From cccedc1aa43f94ce4f8fa6f807a5301250c2213c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 22 Sep 2015 21:52:41 +0600 Subject: [PATCH 1382/1424] [voewster] Detect series geo restriction --- youtube_dl/extractor/viewster.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py index 8defc1800..f8f4143c6 100644 --- a/youtube_dl/extractor/viewster.py +++ b/youtube_dl/extractor/viewster.py @@ -3,12 +3,14 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..compat import ( + compat_HTTPError, compat_urllib_request, compat_urllib_parse, compat_urllib_parse_unquote, ) from ..utils import ( determine_ext, + ExtractorError, int_or_none, parse_iso8601, HEADRequest, @@ -86,9 +88,15 @@ class ViewsterIE(InfoExtractor): # unfinished serie has no Type if info.get('Type') in ['Serie', None]: - episodes = self._download_json( - 'https://public-api.viewster.com/series/%s/episodes' % entry_id, - video_id, 'Downloading series JSON') + try: + episodes = self._download_json( + 'https://public-api.viewster.com/series/%s/episodes' % entry_id, + video_id, 'Downloading series JSON') + except ExtractorError as e: + if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404: + self.raise_geo_restricted() + else: + raise entries = [ self.url_result( 'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster') From 9612f2339909f958f0c7ee7ccc370151ec28358d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 22 Sep 2015 21:54:32 +0600 Subject: [PATCH 1383/1424] [viewster] Detect video geo restriction --- youtube_dl/extractor/viewster.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py index f8f4143c6..7c4328efb 100644 --- a/youtube_dl/extractor/viewster.py +++ b/youtube_dl/extractor/viewster.py @@ -140,6 +140,10 @@ class ViewsterIE(InfoExtractor): f['height'] = int_or_none(self._search_regex( r'^(\d+)[pP]$', format_id, 'height', default=None)) formats.append(f) + + if not formats and not info.get('LanguageSets') and not info.get('VODSettings'): + self.raise_geo_restricted() + self._sort_formats(formats) synopsis = info.get('Synopsis', {}) From 7ce50a355c3a4cbd61f18fb16d8175a97f14d510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 22 Sep 2015 21:55:04 +0600 Subject: [PATCH 1384/1424] [viewster] Add geo restricted tests --- youtube_dl/extractor/viewster.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py index 7c4328efb..487f2d744 100644 --- a/youtube_dl/extractor/viewster.py +++ b/youtube_dl/extractor/viewster.py @@ -63,6 +63,14 @@ class ViewsterIE(InfoExtractor): 'description': 'md5:e7097a8fc97151e25f085c9eb7a1cdb1', }, 'playlist_mincount': 16, + }, { + # geo restricted series + 'url': 'https://www.viewster.com/serie/1280-18794-002/', + 'only_matching': True, + }, { + # geo restricted video + 'url': 'https://www.viewster.com/serie/1280-18794-002/what-is-extraterritoriality-lawo/', + 'only_matching': True, }] _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01' From d0fed4ac028baa40b18082fd813ed93cd658ebe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 22 Sep 2015 22:00:50 +0600 Subject: [PATCH 1385/1424] [viewster] Use tuple --- youtube_dl/extractor/viewster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py index 487f2d744..632e57fb4 100644 --- a/youtube_dl/extractor/viewster.py +++ b/youtube_dl/extractor/viewster.py @@ -95,7 +95,7 @@ class ViewsterIE(InfoExtractor): entry_id = info.get('Id') or info['id'] # unfinished serie has no Type - if info.get('Type') in ['Serie', None]: + if info.get('Type') in ('Serie', None): try: episodes = self._download_json( 'https://public-api.viewster.com/series/%s/episodes' % entry_id, From de3fc356e1597b31c8f0a55a7bf7e201ae436c66 Mon Sep 17 00:00:00 2001 From: remitamine Date: Mon, 21 Sep 2015 14:01:12 +0100 Subject: [PATCH 1386/1424] [ninegag] fix _VALID_URL regex and handle the use of other external providers --- youtube_dl/extractor/ninegag.py | 43 ++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index 7f842b5c2..eee65873a 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -9,17 +9,12 @@ from ..utils import str_to_int class NineGagIE(InfoExtractor): IE_NAME = '9gag' - _VALID_URL = r'''(?x)^https?://(?:www\.)?9gag\.tv/ - (?: - v/(?P[0-9]+)| - p/(?P[a-zA-Z0-9]+)/(?P[^?#/]+) - ) - ''' + _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P[a-zA-Z0-9]+)/(?P[^?#/]+)' _TESTS = [{ - "url": "http://9gag.tv/v/1912", + "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome", "info_dict": { - "id": "1912", + "id": "Kk2X5", "ext": "mp4", "description": "This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)", "title": "\"People Are Awesome 2013\" Is Absolutely Awesome", @@ -31,7 +26,7 @@ class NineGagIE(InfoExtractor): }, 'add_ie': ['Youtube'] }, { - 'url': 'http://9gag.tv/p/KklwM/alternate-banned-opening-scene-of-gravity?ref=fsidebar', + 'url': 'http://9gag.com/tv/p/KklwM/alternate-banned-opening-scene-of-gravity?ref=fsidebar', 'info_dict': { 'id': 'KklwM', 'ext': 'mp4', @@ -42,19 +37,39 @@ class NineGagIE(InfoExtractor): 'upload_date': '20140401', 'uploader_id': 'krishnashenoi93', }, + 'add_ie': ['Youtube'] }] + _EXTERNAL_VIDEO_PROVIDER = { + '1': { + 'url': '%s', + 'ie_key': 'Youtube', + }, + '2': { + 'url': 'http://player.vimeo.com/video/%s', + 'ie_key': 'Vimeo', + }, + '3': { + 'url': 'http://instagram.com/p/%s', + 'ie_key': 'Instagram', + }, + '4': { + 'url': 'http://vine.co/v/%s', + 'ie_key': 'Vine', + }, + } def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('numid') or mobj.group('id') - display_id = mobj.group('display_id') or video_id + video_id = mobj.group('id') + display_id = mobj.group('display_id') webpage = self._download_webpage(url, display_id) post_view = json.loads(self._html_search_regex( r'var postView = new app\.PostView\({\s*post:\s*({.+?}),\s*posts:\s*prefetchedCurrentPost', webpage, 'post view')) - youtube_id = post_view['videoExternalId'] + external_video_id = post_view['videoExternalId'] + external_video_provider = post_view['videoExternalProvider'] title = post_view['title'] description = post_view['description'] view_count = str_to_int(post_view['externalView']) @@ -62,8 +77,8 @@ class NineGagIE(InfoExtractor): return { '_type': 'url_transparent', - 'url': youtube_id, - 'ie_key': 'Youtube', + 'url': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id, + 'ie_key': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key'], 'id': video_id, 'display_id': display_id, 'title': title, From 6b8ce312e3cb7d81e949cd5c64ad2a824d27830b Mon Sep 17 00:00:00 2001 From: remitamine Date: Tue, 22 Sep 2015 19:20:18 +0100 Subject: [PATCH 1387/1424] [ninegag] extract source url --- youtube_dl/extractor/ninegag.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index eee65873a..00bf98ef5 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -68,8 +68,13 @@ class NineGagIE(InfoExtractor): post_view = json.loads(self._html_search_regex( r'var postView = new app\.PostView\({\s*post:\s*({.+?}),\s*posts:\s*prefetchedCurrentPost', webpage, 'post view')) - external_video_id = post_view['videoExternalId'] - external_video_provider = post_view['videoExternalProvider'] + ie_key = None + source_url = post_view.get('sourceUrl') + if not source_url or source_url == '': + external_video_id = post_view['videoExternalId'] + external_video_provider = post_view['videoExternalProvider'] + source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id + ie_key = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key'] title = post_view['title'] description = post_view['description'] view_count = str_to_int(post_view['externalView']) @@ -77,8 +82,8 @@ class NineGagIE(InfoExtractor): return { '_type': 'url_transparent', - 'url': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id, - 'ie_key': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key'], + 'url': source_url, + 'ie_key': ie_key, 'id': video_id, 'display_id': display_id, 'title': title, From da9f18083596d0132d12652acd0bd8983c70c058 Mon Sep 17 00:00:00 2001 From: remitamine Date: Tue, 22 Sep 2015 20:28:00 +0100 Subject: [PATCH 1388/1424] [ninegag] remove unnecessary condition --- youtube_dl/extractor/ninegag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index 00bf98ef5..0a2725c65 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -70,7 +70,7 @@ class NineGagIE(InfoExtractor): ie_key = None source_url = post_view.get('sourceUrl') - if not source_url or source_url == '': + if not source_url: external_video_id = post_view['videoExternalId'] external_video_provider = post_view['videoExternalProvider'] source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id From e28c794699596912092635014b041d0af888fd08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 23 Sep 2015 01:40:06 +0600 Subject: [PATCH 1389/1424] [9gag] Make display_id optional --- youtube_dl/extractor/ninegag.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index 0a2725c65..6103c7517 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -9,7 +9,7 @@ from ..utils import str_to_int class NineGagIE(InfoExtractor): IE_NAME = '9gag' - _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P[a-zA-Z0-9]+)/(?P[^?#/]+)' + _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P[a-zA-Z0-9]+)(?:/(?P[^?#/]+))?' _TESTS = [{ "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome", @@ -61,7 +61,7 @@ class NineGagIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') - display_id = mobj.group('display_id') + display_id = mobj.group('display_id') or video_id webpage = self._download_webpage(url, display_id) From c3a4e2ec40f9a90d4ae8991e6764641c744ec8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 23 Sep 2015 01:41:44 +0600 Subject: [PATCH 1390/1424] [9gag] Remove redundant test --- youtube_dl/extractor/ninegag.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index 6103c7517..34879d1c8 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -26,18 +26,8 @@ class NineGagIE(InfoExtractor): }, 'add_ie': ['Youtube'] }, { - 'url': 'http://9gag.com/tv/p/KklwM/alternate-banned-opening-scene-of-gravity?ref=fsidebar', - 'info_dict': { - 'id': 'KklwM', - 'ext': 'mp4', - 'display_id': 'alternate-banned-opening-scene-of-gravity', - "description": "While Gravity was a pretty awesome movie already, YouTuber Krishna Shenoi came up with a way to improve upon it, introducing a much better solution to Sandra Bullock's seemingly endless tumble in space. The ending is priceless.", - 'title': "Banned Opening Scene Of \"Gravity\" That Changes The Whole Movie", - 'uploader': 'Krishna Shenoi', - 'upload_date': '20140401', - 'uploader_id': 'krishnashenoi93', - }, - 'add_ie': ['Youtube'] + 'url': 'http://9gag.com/tv/p/KklwM', + 'only_matching': True, }] _EXTERNAL_VIDEO_PROVIDER = { '1': { From 6400f8ec0f20011a7e39da62e0a3e55e0fd2759a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 23 Sep 2015 01:43:26 +0600 Subject: [PATCH 1391/1424] [9gag] Allow old .tv domain There are still references to it in webpage's source --- youtube_dl/extractor/ninegag.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index 34879d1c8..d157b0d10 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -9,7 +9,7 @@ from ..utils import str_to_int class NineGagIE(InfoExtractor): IE_NAME = '9gag' - _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P[a-zA-Z0-9]+)(?:/(?P[^?#/]+))?' + _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/p/(?P[a-zA-Z0-9]+)(?:/(?P[^?#/]+))?' _TESTS = [{ "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome", @@ -28,6 +28,9 @@ class NineGagIE(InfoExtractor): }, { 'url': 'http://9gag.com/tv/p/KklwM', 'only_matching': True, + }, { + 'url': 'http://9gag.tv/p/Kk2X5', + 'only_matching': True, }] _EXTERNAL_VIDEO_PROVIDER = { '1': { From 5600e214c3b63e3e2a0862bea230026c02073d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 23 Sep 2015 01:44:38 +0600 Subject: [PATCH 1392/1424] [9gag] Make post view regex more robust --- youtube_dl/extractor/ninegag.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index d157b0d10..692cc3368 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -59,7 +59,8 @@ class NineGagIE(InfoExtractor): webpage = self._download_webpage(url, display_id) post_view = json.loads(self._html_search_regex( - r'var postView = new app\.PostView\({\s*post:\s*({.+?}),\s*posts:\s*prefetchedCurrentPost', webpage, 'post view')) + r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost', + webpage, 'post view')) ie_key = None source_url = post_view.get('sourceUrl') From 8ca2e93e1ab3485946df19d56a87e56699b2a712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 23 Sep 2015 01:46:40 +0600 Subject: [PATCH 1393/1424] [9gag] Relax optional fields --- youtube_dl/extractor/ninegag.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index 692cc3368..6daae7318 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -70,8 +70,8 @@ class NineGagIE(InfoExtractor): source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id ie_key = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key'] title = post_view['title'] - description = post_view['description'] - view_count = str_to_int(post_view['externalView']) + description = post_view.get('description') + view_count = str_to_int(post_view.get('externalView')) thumbnail = post_view.get('thumbnail_700w') or post_view.get('ogImageUrl') or post_view.get('thumbnail_300w') return { From c659022b5cabebaba3275df2d0f4ae97b468eac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 23 Sep 2015 01:48:13 +0600 Subject: [PATCH 1394/1424] [9gag] Modernize --- youtube_dl/extractor/ninegag.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index 6daae7318..d8e103189 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -58,9 +58,11 @@ class NineGagIE(InfoExtractor): webpage = self._download_webpage(url, display_id) - post_view = json.loads(self._html_search_regex( - r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost', - webpage, 'post view')) + post_view = self._parse_json( + self._search_regex( + r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost', + webpage, 'post view'), + display_id) ie_key = None source_url = post_view.get('sourceUrl') From 8ea6bd2802a0dbb860187bb5aaf687585ff7c1c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 23 Sep 2015 01:55:16 +0600 Subject: [PATCH 1395/1424] [9gag] Add vimeo test --- youtube_dl/extractor/ninegag.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index d8e103189..0ef6a6c65 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import re -import json from .common import InfoExtractor from ..utils import str_to_int @@ -22,9 +21,21 @@ class NineGagIE(InfoExtractor): 'uploader': 'CompilationChannel', 'upload_date': '20131110', "view_count": int, - "thumbnail": "re:^https?://", }, - 'add_ie': ['Youtube'] + 'add_ie': ['Youtube'], + }, { + 'url': 'http://9gag.com/tv/p/aKolP3', + 'info_dict': { + 'id': 'aKolP3', + 'ext': 'mp4', + 'title': 'This Guy Travelled 11 countries In 44 days Just To Make This Amazing Video', + 'description': "I just saw more in 1 minute than I've seen in 1 year. This guy's video is epic!!", + 'uploader_id': 'rickmereki', + 'uploader': 'Rick Mereki', + 'upload_date': '20110803', + 'view_count': int, + }, + 'add_ie': ['Vimeo'], }, { 'url': 'http://9gag.com/tv/p/KklwM', 'only_matching': True, From d8fef8faacad4f3b9d13e20df4ee9344ada3c68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 23 Sep 2015 01:56:26 +0600 Subject: [PATCH 1396/1424] [9gag] Quotes consistency --- youtube_dl/extractor/ninegag.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index 0ef6a6c65..4adff197e 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -11,16 +11,16 @@ class NineGagIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/p/(?P[a-zA-Z0-9]+)(?:/(?P[^?#/]+))?' _TESTS = [{ - "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome", - "info_dict": { - "id": "Kk2X5", - "ext": "mp4", - "description": "This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)", - "title": "\"People Are Awesome 2013\" Is Absolutely Awesome", + 'url': 'http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome', + 'info_dict': { + 'id': 'Kk2X5', + 'ext': 'mp4', + 'description': 'This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)', + 'title': '\"People Are Awesome 2013\" Is Absolutely Awesome', 'uploader_id': 'UCdEH6EjDKwtTe-sO2f0_1XA', 'uploader': 'CompilationChannel', 'upload_date': '20131110', - "view_count": int, + 'view_count': int, }, 'add_ie': ['Youtube'], }, { From 78f9fb902b36c2a12dfe50d9724ac470b03c87d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 23 Sep 2015 01:58:47 +0600 Subject: [PATCH 1397/1424] [9gag] Support embed URLs --- youtube_dl/extractor/ninegag.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index 4adff197e..a06d38afd 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -8,7 +8,7 @@ from ..utils import str_to_int class NineGagIE(InfoExtractor): IE_NAME = '9gag' - _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/p/(?P[a-zA-Z0-9]+)(?:/(?P[^?#/]+))?' + _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/(?:p|embed)/(?P[a-zA-Z0-9]+)(?:/(?P[^?#/]+))?' _TESTS = [{ 'url': 'http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome', @@ -42,7 +42,11 @@ class NineGagIE(InfoExtractor): }, { 'url': 'http://9gag.tv/p/Kk2X5', 'only_matching': True, + }, { + 'url': 'http://9gag.com/tv/embed/a5Dmvl', + 'only_matching': True, }] + _EXTERNAL_VIDEO_PROVIDER = { '1': { 'url': '%s', From b942db3dc3d51db0f24ac98ada861d8e2b3451db Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Tue, 22 Sep 2015 22:41:53 +0200 Subject: [PATCH 1398/1424] release 2015.09.22 --- docs/supportedsites.md | 17 ++++++++++------- youtube_dl/version.py | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 66091e6be..ab153af6b 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -122,7 +122,6 @@ - **defense.gouv.fr** - **DHM**: Filmarchiv - Deutsches Historisches Museum - **Discovery** - - **divxstage**: DivxStage - **Dotsub** - **DouyuTV**: 斗鱼 - **dramafever** @@ -286,7 +285,7 @@ - **Minhateca** - **MinistryGrid** - **miomio.tv** - - **mitele.es** + - **MiTele**: mitele.es - **mixcloud** - **MLB** - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net @@ -317,7 +316,6 @@ - **Myvi** - **myvideo** - **MyVidster** - - **N-JOY** - **n-tv.de** - **NationalGeographic** - **Naver** @@ -326,7 +324,9 @@ - **NBCNews** - **NBCSports** - **NBCSportsVPlayer** - - **ndr**: NDR.de - Mediathek + - **ndr**: NDR.de - Norddeutscher Rundfunk + - **ndr:embed** + - **ndr:embed:base** - **NDTV** - **NerdCubedFeed** - **Nerdist** @@ -349,12 +349,16 @@ - **nhl.com:videocenter**: NHL videocenter category - **niconico**: ニコニコ動画 - **NiconicoPlaylist** + - **njoy**: N-JOY + - **njoy:embed** - **Noco** - **Normalboots** - **NosVideo** - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz - **novamov**: NovaMov - - **Nowness** + - **nowness** + - **nowness:playlist** + - **nowness:series** - **NowTV** - **nowvideo**: NowVideo - **npo**: npo.nl and ntr.nl @@ -375,7 +379,6 @@ - **OnionStudios** - **Ooyala** - **OoyalaExternal** - - **OpenFilm** - **orf:fm4**: radio FM4 - **orf:iptv**: iptv.ORF.at - **orf:oe1**: Radio Österreich 1 @@ -530,7 +533,7 @@ - **techtv.mit.edu** - **ted** - **TeleBruxelles** - - **telecinco.es** + - **Telecinco**: telecinco.es, cuatro.com and mediaset.es - **Telegraaf** - **TeleMB** - **TeleTask** diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 0cc7411f2..7ef4f2755 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2015.09.09' +__version__ = '2015.09.22' From f1028194636e7acafb29fb38244cc7b1347d9313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 23 Sep 2015 02:46:24 +0600 Subject: [PATCH 1399/1424] [downloader/hls] Pass http headers to downloader --- youtube_dl/downloader/hls.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py index 7743e176a..a62d2047b 100644 --- a/youtube_dl/downloader/hls.py +++ b/youtube_dl/downloader/hls.py @@ -28,9 +28,18 @@ class HlsFD(FileDownloader): return False ffpp.check_version() - args = [ - encodeArgument(opt) - for opt in (ffpp.executable, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')] + args = [ffpp.executable, '-y'] + + if info_dict['http_headers']: + # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv: + # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header. + args += [ + '-headers', + ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items())] + + args += ['-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc'] + + args = [encodeArgument(opt) for opt in args] args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True)) self._debug_cmd(args) From eb11cbe8674705647d6bd2947a44e08543663633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 23 Sep 2015 19:54:40 +0600 Subject: [PATCH 1400/1424] [soundcloud] Update client id (Closes #6930) --- youtube_dl/extractor/soundcloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index ed5dcc0d3..2b60d354a 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -113,7 +113,7 @@ class SoundcloudIE(InfoExtractor): }, ] - _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28' + _CLIENT_ID = '02gUJC0hH2ct1EGOcYXQIzRFU91c72Ea' _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf' def report_resolve(self, video_id): From 57565375c85ff6adb11beb961fd61d6bdd023ec1 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 23 Sep 2015 22:22:04 +0800 Subject: [PATCH 1401/1424] [iqiyi] Fix extraction (fixes #6878) --- youtube_dl/extractor/iqiyi.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py index 393e67e35..2ce6627a4 100644 --- a/youtube_dl/extractor/iqiyi.py +++ b/youtube_dl/extractor/iqiyi.py @@ -95,6 +95,10 @@ class IqiyiIE(InfoExtractor): ('10', 'h1'), ] + @staticmethod + def md5_text(text): + return hashlib.md5(text.encode('utf-8')).hexdigest() + def construct_video_urls(self, data, video_id, _uuid): def do_xor(x, y): a = y % 3 @@ -179,6 +183,7 @@ class IqiyiIE(InfoExtractor): def get_raw_data(self, tvid, video_id, enc_key, _uuid): tm = str(int(time.time())) + tail = tm + tvid param = { 'key': 'fvip', 'src': hashlib.md5(b'youtube-dl').hexdigest(), @@ -186,13 +191,11 @@ class IqiyiIE(InfoExtractor): 'vid': video_id, 'vinfo': 1, 'tm': tm, - 'enc': hashlib.md5( - (enc_key + tm + tvid).encode('utf8')).hexdigest(), + 'enc': self.md5_text((enc_key + tail)[1:64:2] + tail), 'qyid': _uuid, 'tn': random.random(), 'um': 0, - 'authkey': hashlib.md5( - (tm + tvid).encode('utf8')).hexdigest() + 'authkey': self.md5_text(self.md5_text('') + tail), } api_url = 'http://cache.video.qiyi.com/vms' + '?' + \ @@ -201,7 +204,8 @@ class IqiyiIE(InfoExtractor): return raw_data def get_enc_key(self, swf_url, video_id): - enc_key = '3601ba290e4f4662848c710e2122007e' # last update at 2015-08-10 for Zombie + # TODO: automatic key extraction + enc_key = 'eac64f22daf001da6ba9aa8da4d501508bbe90a4d4091fea3b0582a85b38c2cc' # last update at 2015-09-23-23 for Zombie::bite return enc_key def _real_extract(self, url): From 19f93d906e29e9a505d4bf5d286d75224c342c37 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 23 Sep 2015 22:25:16 +0800 Subject: [PATCH 1402/1424] [iqiyi] Use md5_text for all MD5 calls --- youtube_dl/extractor/iqiyi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py index 2ce6627a4..ce1ab3820 100644 --- a/youtube_dl/extractor/iqiyi.py +++ b/youtube_dl/extractor/iqiyi.py @@ -125,7 +125,7 @@ class IqiyiIE(InfoExtractor): note='Download path key of segment %d for format %s' % (segment_index + 1, format_id) )['t'] t = str(int(math.floor(int(tm) / (600.0)))) - return hashlib.md5((t + mg + x).encode('utf8')).hexdigest() + return self.md5_text(t + mg + x) video_urls_dict = {} for format_item in data['vp']['tkl'][0]['vs']: @@ -186,7 +186,7 @@ class IqiyiIE(InfoExtractor): tail = tm + tvid param = { 'key': 'fvip', - 'src': hashlib.md5(b'youtube-dl').hexdigest(), + 'src': self.md5_text('youtube-dl'), 'tvId': tvid, 'vid': video_id, 'vinfo': 1, From 4395ca2e04f38c110259270fa4dfc4d9814aa926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 24 Sep 2015 19:56:54 +0600 Subject: [PATCH 1403/1424] [xhamster] Fix title extraction (Closes #6944) --- youtube_dl/extractor/xhamster.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 97315750f..f12fe13b1 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -63,7 +63,9 @@ class XHamsterIE(InfoExtractor): mrss_url = '%s://xhamster.com/movies/%s/%s.html' % (proto, video_id, seo) webpage = self._download_webpage(mrss_url, video_id) - title = self._html_search_regex(r'(?P<title>.+?) - xHamster\.com', webpage, 'title') + title = self._html_search_regex( + [r'(?P<title>.+?)(?:, Free Porn: xHamster| - xHamster\.com)', + r'

([^<]+)

'], webpage, 'title') # Only a few videos have an description mobj = re.search(r'Description: ([^<]+)', webpage) From 05b476a27087a45b8d418607123d8db62bf1770f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 24 Sep 2015 23:38:53 +0600 Subject: [PATCH 1404/1424] [vidme] Prefer non clip (Closes #6924) --- youtube_dl/extractor/vidme.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py index 9a794e609..078d283b2 100644 --- a/youtube_dl/extractor/vidme.py +++ b/youtube_dl/extractor/vidme.py @@ -119,6 +119,7 @@ class VidmeIE(InfoExtractor): 'url': f['uri'], 'width': int_or_none(f.get('width')), 'height': int_or_none(f.get('height')), + 'preference': 0 if f.get('type', '').endswith('clip') else 1, } for f in video.get('formats', []) if f.get('uri')] self._sort_formats(formats) From 9fbd4b35a27a83055c3e170ab32f2b3e56f9616e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 24 Sep 2015 23:48:23 +0600 Subject: [PATCH 1405/1424] [nhl] Add support for embedded URLs --- youtube_dl/extractor/nhl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py index 279b18386..970a11f7c 100644 --- a/youtube_dl/extractor/nhl.py +++ b/youtube_dl/extractor/nhl.py @@ -72,7 +72,7 @@ class NHLBaseInfoExtractor(InfoExtractor): class NHLIE(NHLBaseInfoExtractor): IE_NAME = 'nhl.com' - _VALID_URL = r'https?://video(?P\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)(?:id|hlg)=(?P[-0-9a-zA-Z,]+)' + _VALID_URL = r'https?://video(?P\.[^.]*)?\.nhl\.com/videocenter/(?:console|embed)?(?:\?(?:.*?[?&])?)(?:id|hlg|playlist)=(?P[-0-9a-zA-Z,]+)' _TESTS = [{ 'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614', @@ -136,6 +136,9 @@ class NHLIE(NHLBaseInfoExtractor): 'params': { 'skip_download': True, # Requires rtmpdump } + }, { + 'url': 'http://video.nhl.com/videocenter/embed?playlist=836127', + 'only_matching': True, }] def _real_extract(self, url): From 9c58885c70af75655288933220a99b4c4215ab4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 24 Sep 2015 23:54:16 +0600 Subject: [PATCH 1406/1424] [nhl:news] Add support for iframe embeds (Closes #6941) --- youtube_dl/extractor/nhl.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py index 970a11f7c..e98a5ef89 100644 --- a/youtube_dl/extractor/nhl.py +++ b/youtube_dl/extractor/nhl.py @@ -149,9 +149,9 @@ class NHLIE(NHLBaseInfoExtractor): class NHLNewsIE(NHLBaseInfoExtractor): IE_NAME = 'nhl.com:news' IE_DESC = 'NHL news' - _VALID_URL = r'https?://(?:www\.)?nhl\.com/ice/news\.html?(?:\?(?:.*?[?&])?)id=(?P[-0-9a-zA-Z]+)' + _VALID_URL = r'https?://(?:.+?\.)?nhl\.com/(?:ice|club)/news\.html?(?:\?(?:.*?[?&])?)id=(?P[-0-9a-zA-Z]+)' - _TEST = { + _TESTS = [{ 'url': 'http://www.nhl.com/ice/news.htm?id=750727', 'md5': '4b3d1262e177687a3009937bd9ec0be8', 'info_dict': { @@ -162,13 +162,26 @@ class NHLNewsIE(NHLBaseInfoExtractor): 'duration': 37, 'upload_date': '20150128', }, - } + }, { + # iframe embed + 'url': 'http://sabres.nhl.com/club/news.htm?id=780189', + 'md5': '9f663d1c006c90ac9fb82777d4294e12', + 'info_dict': { + 'id': '836127', + 'ext': 'mp4', + 'title': 'Morning Skate: OTT vs. BUF (9/23/15)', + 'description': "Brian Duff chats with Tyler Ennis prior to Buffalo's first preseason home game.", + 'duration': 93, + 'upload_date': '20150923', + }, + }] def _real_extract(self, url): news_id = self._match_id(url) webpage = self._download_webpage(url, news_id) video_id = self._search_regex( - [r'pVid(\d+)', r"nlid\s*:\s*'(\d+)'"], + [r'pVid(\d+)', r"nlid\s*:\s*'(\d+)'", + r']+src=["\']https?://video.*?\.nhl\.com/videocenter/embed\?.*\bplaylist=(\d+)'], webpage, 'video id') return self._real_extract_video(video_id) From 47024eb564fd4047e362680f0a68304d1df79495 Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 24 Sep 2015 19:49:10 +0100 Subject: [PATCH 1407/1424] [hostingbulk] remove extractor --- youtube_dl/extractor/__init__.py | 1 - youtube_dl/extractor/hostingbulk.py | 80 ----------------------------- 2 files changed, 81 deletions(-) delete mode 100644 youtube_dl/extractor/hostingbulk.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index c16e9e3d2..7272859db 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -228,7 +228,6 @@ from .historicfilms import HistoricFilmsIE from .history import HistoryIE from .hitbox import HitboxIE, HitboxLiveIE from .hornbunny import HornBunnyIE -from .hostingbulk import HostingBulkIE from .hotnewhiphop import HotNewHipHopIE from .howcast import HowcastIE from .howstuffworks import HowStuffWorksIE diff --git a/youtube_dl/extractor/hostingbulk.py b/youtube_dl/extractor/hostingbulk.py deleted file mode 100644 index a3154cfde..000000000 --- a/youtube_dl/extractor/hostingbulk.py +++ /dev/null @@ -1,80 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor -from ..compat import ( - compat_urllib_request, -) -from ..utils import ( - ExtractorError, - int_or_none, - urlencode_postdata, -) - - -class HostingBulkIE(InfoExtractor): - _VALID_URL = r'''(?x) - https?://(?:www\.)?hostingbulk\.com/ - (?:embed-)?(?P[A-Za-z0-9]{12})(?:-\d+x\d+)?\.html''' - _FILE_DELETED_REGEX = r'File Not Found' - _TEST = { - 'url': 'http://hostingbulk.com/n0ulw1hv20fm.html', - 'md5': '6c8653c8ecf7ebfa83b76e24b7b2fe3f', - 'info_dict': { - 'id': 'n0ulw1hv20fm', - 'ext': 'mp4', - 'title': 'md5:5afeba33f48ec87219c269e054afd622', - 'filesize': 6816081, - 'thumbnail': 're:^http://.*\.jpg$', - } - } - - def _real_extract(self, url): - video_id = self._match_id(url) - url = 'http://hostingbulk.com/{0:}.html'.format(video_id) - - # Custom request with cookie to set language to English, so our file - # deleted regex would work. - request = compat_urllib_request.Request( - url, headers={'Cookie': 'lang=english'}) - webpage = self._download_webpage(request, video_id) - - if re.search(self._FILE_DELETED_REGEX, webpage) is not None: - raise ExtractorError('Video %s does not exist' % video_id, - expected=True) - - title = self._html_search_regex(r'

(.*?)

', webpage, 'title') - filesize = int_or_none( - self._search_regex( - r'\((\d+)\sbytes?\)', - webpage, - 'filesize', - fatal=False - ) - ) - thumbnail = self._search_regex( - r' Date: Thu, 24 Sep 2015 23:05:32 +0200 Subject: [PATCH 1408/1424] More title extraction fixing. --- youtube_dl/extractor/xhamster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index f12fe13b1..8938c0e45 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -64,7 +64,7 @@ class XHamsterIE(InfoExtractor): webpage = self._download_webpage(mrss_url, video_id) title = self._html_search_regex( - [r'(?P<title>.+?)(?:, Free Porn: xHamster| - xHamster\.com)', + [r'(?P<title>.+?)(?:, (?:[^,]+? )?Porn: xHamster| - xHamster\.com)', r'

([^<]+)

'], webpage, 'title') # Only a few videos have an description From 857421024daf810e92036149cc02bcf1c337da5c Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 24 Sep 2015 21:55:44 +0100 Subject: [PATCH 1409/1424] [iconosquare] fix info extraction --- youtube_dl/extractor/iconosquare.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py index 70e4c0d41..4fff8c0b3 100644 --- a/youtube_dl/extractor/iconosquare.py +++ b/youtube_dl/extractor/iconosquare.py @@ -1,7 +1,10 @@ from __future__ import unicode_literals from .common import InfoExtractor -from ..utils import int_or_none +from ..utils import ( + int_or_none, + get_element_by_id, +) class IconosquareIE(InfoExtractor): @@ -12,7 +15,7 @@ class IconosquareIE(InfoExtractor): 'info_dict': { 'id': '522207370455279102_24101272', 'ext': 'mp4', - 'title': 'Instagram media by @aguynamedpatrick (Patrick Janelle)', + 'title': 'A little over a year ago, I posted my first #dailycortado, a drink introduced to...', 'description': 'md5:644406a9ec27457ed7aa7a9ebcd4ce3d', 'timestamp': 1376471991, 'upload_date': '20130814', @@ -29,8 +32,7 @@ class IconosquareIE(InfoExtractor): webpage = self._download_webpage(url, video_id) media = self._parse_json( - self._search_regex( - r'window\.media\s*=\s*({.+?});\n', webpage, 'media'), + get_element_by_id('mediaJson', webpage), video_id) formats = [{ @@ -42,7 +44,7 @@ class IconosquareIE(InfoExtractor): self._sort_formats(formats) title = self._html_search_regex( - r'(.+?)(?: *\(Videos?\))? \| (?:Iconosquare|Statigram)', + r'(.+?)', webpage, 'title') timestamp = int_or_none(media.get('created_time') or media.get('caption', {}).get('created_time')) From d4364f30bd39528b4da487799380737c330e88c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 25 Sep 2015 04:44:52 +0600 Subject: [PATCH 1410/1424] [iconosquare] Revert title (Closes #6954) --- youtube_dl/extractor/iconosquare.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py index 4fff8c0b3..bb69c7a0f 100644 --- a/youtube_dl/extractor/iconosquare.py +++ b/youtube_dl/extractor/iconosquare.py @@ -4,6 +4,7 @@ from .common import InfoExtractor from ..utils import ( int_or_none, get_element_by_id, + remove_end, ) @@ -15,7 +16,7 @@ class IconosquareIE(InfoExtractor): 'info_dict': { 'id': '522207370455279102_24101272', 'ext': 'mp4', - 'title': 'A little over a year ago, I posted my first #dailycortado, a drink introduced to...', + 'title': 'Instagram photo by @aguynamedpatrick (Patrick Janelle)', 'description': 'md5:644406a9ec27457ed7aa7a9ebcd4ce3d', 'timestamp': 1376471991, 'upload_date': '20130814', @@ -43,9 +44,7 @@ class IconosquareIE(InfoExtractor): } for format_id, f in media['videos'].items()] self._sort_formats(formats) - title = self._html_search_regex( - r'(.+?)', - webpage, 'title') + title = remove_end(self._og_search_title(webpage), ' - via Iconosquare') timestamp = int_or_none(media.get('created_time') or media.get('caption', {}).get('created_time')) description = media.get('caption', {}).get('text') From 9b166fc1f8039fe3f3632c40848ce590ade9f3fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 25 Sep 2015 04:45:31 +0600 Subject: [PATCH 1411/1424] [iconosquare] Extract comments --- youtube_dl/extractor/iconosquare.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py index bb69c7a0f..a39f422e9 100644 --- a/youtube_dl/extractor/iconosquare.py +++ b/youtube_dl/extractor/iconosquare.py @@ -62,6 +62,14 @@ class IconosquareIE(InfoExtractor): 'height': int_or_none(t.get('height')) } for thumbnail_id, t in media.get('images', {}).items()] + comments = [{ + 'id': comment.get('id'), + 'text': comment['text'], + 'timestamp': int_or_none(comment.get('created_time')), + 'author': comment.get('from', {}).get('full_name'), + 'author_id': comment.get('from', {}).get('username'), + } for comment in media.get('comments', {}).get('data', []) if 'text' in comment] + return { 'id': video_id, 'title': title, @@ -73,4 +81,5 @@ class IconosquareIE(InfoExtractor): 'comment_count': comment_count, 'like_count': like_count, 'formats': formats, + 'comments': comments, } From 882fc9052e310b5ac6675488bba767c43ca2185e Mon Sep 17 00:00:00 2001 From: remitamine Date: Wed, 23 Sep 2015 17:48:39 +0100 Subject: [PATCH 1412/1424] [condenast] fix extraction and add support for other sites --- youtube_dl/extractor/condenast.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py index 3db4db4e4..22c66da26 100644 --- a/youtube_dl/extractor/condenast.py +++ b/youtube_dl/extractor/condenast.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals import re -import json from .common import InfoExtractor from ..compat import ( @@ -24,13 +23,25 @@ class CondeNastIE(InfoExtractor): # The keys are the supported sites and the values are the name to be shown # to the user and in the extractor description. _SITES = { - 'wired': 'WIRED', - 'gq': 'GQ', - 'vogue': 'Vogue', - 'glamour': 'Glamour', - 'wmagazine': 'W Magazine', - 'vanityfair': 'Vanity Fair', + 'allure': 'Allure', + 'architecturaldigest': 'Architectural Digest', + 'arstechnica': 'Ars Technica', + 'bonappetit': 'Bon Appetit', + 'brides': 'Brides', 'cnevids': 'Condé Nast', + 'cntraveler': 'Condé Nast Traveler', + 'details': 'Details', + 'epicurious': 'Epicurious', + 'glamour': 'Glamour', + 'golfdigest': 'Golf Digest', + 'gq': 'GQ', + 'newyorker': 'The New Yorker', + 'self': 'SELF', + 'teenvogue': 'Teen Vogue', + 'vanityfair': 'Vanity Fair', + 'vogue': 'Vogue', + 'wired': 'WIRED', + 'wmagazine': 'W Magazine', } _VALID_URL = r'http://(video|www|player)\.(?P%s)\.com/(?Pwatch|series|video|embed)/(?P[^/?#]+)' % '|'.join(_SITES.keys()) @@ -86,8 +97,8 @@ class CondeNastIE(InfoExtractor): info_url = base_info_url + data info_page = self._download_webpage(info_url, video_id, 'Downloading video info') - video_info = self._search_regex(r'var video = ({.+?});', info_page, 'video info') - video_info = json.loads(video_info) + video_info = self._search_regex(r'var\s*video\s*=\s*({.+?});', info_page, 'video info') + video_info = self._parse_json(video_info, video_id) formats = [{ 'format_id': '%s-%s' % (fdata['type'].split('/')[-1], fdata['quality']), From 2949a6cda9c39b3ff32891bdf0d6b48c46973f82 Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 24 Sep 2015 15:54:23 +0100 Subject: [PATCH 1413/1424] [condenast] fix video info regex --- youtube_dl/extractor/condenast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py index 22c66da26..d1380ac8d 100644 --- a/youtube_dl/extractor/condenast.py +++ b/youtube_dl/extractor/condenast.py @@ -97,7 +97,7 @@ class CondeNastIE(InfoExtractor): info_url = base_info_url + data info_page = self._download_webpage(info_url, video_id, 'Downloading video info') - video_info = self._search_regex(r'var\s*video\s*=\s*({.+?});', info_page, 'video info') + video_info = self._search_regex(r'var\s+video\s*=\s*({.+?});', info_page, 'video info') video_info = self._parse_json(video_info, video_id) formats = [{ From 42ca72dff38c6cb23724dd91b39550e805bd8d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 25 Sep 2015 05:15:21 +0600 Subject: [PATCH 1414/1424] [condenast] Keep acute accent --- youtube_dl/extractor/condenast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py index d1380ac8d..ef1d28091 100644 --- a/youtube_dl/extractor/condenast.py +++ b/youtube_dl/extractor/condenast.py @@ -26,7 +26,7 @@ class CondeNastIE(InfoExtractor): 'allure': 'Allure', 'architecturaldigest': 'Architectural Digest', 'arstechnica': 'Ars Technica', - 'bonappetit': 'Bon Appetit', + 'bonappetit': 'Bon Appétit', 'brides': 'Brides', 'cnevids': 'Condé Nast', 'cntraveler': 'Condé Nast Traveler', From 0940c5b4c6a068d4919fd29a8db2a85ab3bbf703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 25 Sep 2015 05:18:45 +0600 Subject: [PATCH 1415/1424] [condenast] Do not capture unused group in _VALID_URL --- youtube_dl/extractor/condenast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py index ef1d28091..d6949ca28 100644 --- a/youtube_dl/extractor/condenast.py +++ b/youtube_dl/extractor/condenast.py @@ -44,7 +44,7 @@ class CondeNastIE(InfoExtractor): 'wmagazine': 'W Magazine', } - _VALID_URL = r'http://(video|www|player)\.(?P%s)\.com/(?Pwatch|series|video|embed)/(?P[^/?#]+)' % '|'.join(_SITES.keys()) + _VALID_URL = r'http://(?:video|www|player)\.(?P%s)\.com/(?Pwatch|series|video|embed)/(?P[^/?#]+)' % '|'.join(_SITES.keys()) IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values())) EMBED_URL = r'(?:https?:)?//player\.(?P%s)\.com/(?Pembed)/.+?' % '|'.join(_SITES.keys()) From 2e40a12225fbe64b84b3975b3063a676df0f4522 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 17:24:35 +0800 Subject: [PATCH 1416/1424] [fktv] Correct spellings --- youtube_dl/extractor/fktv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index c2aa23aa2..93c4fd641 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -28,9 +28,9 @@ class FKTVIE(InfoExtractor): webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode) title = clean_html(self._html_search_regex('

([^<]+?)

', webpage, 'title')) - matchs = re.search(r'(?s)]*poster="([^"]+)"[^>]*>(.*?)', webpage) - if matchs: - poster, sources = matchs.groups() + matches = re.search(r'(?s)]*poster="([^"]+)"[^>]*>(.*?)', webpage) + if matches: + poster, sources = matches.groups() urls = re.findall(r'(?s)]*src="([^"]+)"[^>]*>', sources) if sources: formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] From 8ddf48d59f2d04b7411202eb6bf02c6eaa387035 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 17:48:51 +0800 Subject: [PATCH 1417/1424] [fktv] Raise an error is no videos found --- youtube_dl/extractor/fktv.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index 93c4fd641..74c6cf866 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -6,6 +6,7 @@ from .common import InfoExtractor from ..utils import ( clean_html, determine_ext, + ExtractorError, ) @@ -29,14 +30,15 @@ class FKTVIE(InfoExtractor): webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode) title = clean_html(self._html_search_regex('

([^<]+?)

', webpage, 'title')) matches = re.search(r'(?s)]*poster="([^"]+)"[^>]*>(.*?)', webpage) - if matches: - poster, sources = matches.groups() - urls = re.findall(r'(?s)]*src="([^"]+)"[^>]*>', sources) - if sources: - formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] - return { - 'id': episode, - 'title': title, - 'formats': formats, - 'thumbnail': poster, - } + if matches is None: + raise ExtractorError('Unable to extract the video') + + poster, sources = matches.groups() + urls = re.findall(r'(?s)]*src="([^"]+)"[^>]*>', sources) + formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] + return { + 'id': episode, + 'title': title, + 'formats': formats, + 'thumbnail': poster, + } From 140359fc2cfe7e9cbfecfd2fd625c6407232fe0f Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 17:51:48 +0800 Subject: [PATCH 1418/1424] [fktv] Correct and improve some regexs --- youtube_dl/extractor/fktv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index 74c6cf866..b081eb535 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -28,13 +28,13 @@ class FKTVIE(InfoExtractor): episode = self._match_id(url) webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode) - title = clean_html(self._html_search_regex('

([^<]+?)

', webpage, 'title')) - matches = re.search(r'(?s)]*poster="([^"]+)"[^>]*>(.*?)', webpage) + title = clean_html(self._html_search_regex('

([^<]+)

', webpage, 'title')) + matches = re.search(r'(?s)]+poster="([^"]+)"[^>]*>(.*)', webpage) if matches is None: raise ExtractorError('Unable to extract the video') poster, sources = matches.groups() - urls = re.findall(r'(?s)]*src="([^"]+)"[^>]*>', sources) + urls = re.findall(r']+src="([^"]+)"', sources) formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] return { 'id': episode, From 577380396171ba096240bfb3101f8151e32b587a Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 17:58:44 +0800 Subject: [PATCH 1419/1424] [fktv] Correct thumbnail extraction and add the test --- youtube_dl/extractor/fktv.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index b081eb535..289cbb8c8 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -21,6 +21,7 @@ class FKTVIE(InfoExtractor): 'id': '1', 'ext': 'mp4', 'title': 'Folge 1 vom 10. April 2007', + 'thumbnail': 're:^https?://.*\.jpg$', }, } @@ -29,11 +30,14 @@ class FKTVIE(InfoExtractor): webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode) title = clean_html(self._html_search_regex('

([^<]+)

', webpage, 'title')) - matches = re.search(r'(?s)]+poster="([^"]+)"[^>]*>(.*)', webpage) + matches = re.search(r'(?s)]+(?:poster="([^"]+)")?[^>]*>(.*)', webpage) if matches is None: raise ExtractorError('Unable to extract the video') poster, sources = matches.groups() + if poster is None: + self.report_warning('unable to extract thumbnail') + urls = re.findall(r']+src="([^"]+)"', sources) formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] return { From 711762f0b70b65f2f28f2cd8023497f50fccd81a Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 18:01:08 +0800 Subject: [PATCH 1420/1424] [fktv] Coding style --- youtube_dl/extractor/fktv.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index 289cbb8c8..0c14834f9 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -28,9 +28,12 @@ class FKTVIE(InfoExtractor): def _real_extract(self, url): episode = self._match_id(url) - webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode) - title = clean_html(self._html_search_regex('

([^<]+)

', webpage, 'title')) - matches = re.search(r'(?s)]+(?:poster="([^"]+)")?[^>]*>(.*)', webpage) + webpage = self._download_webpage( + 'http://fernsehkritik.tv/folge-%s/play' % episode, episode) + title = clean_html(self._html_search_regex( + '

([^<]+)

', webpage, 'title')) + matches = re.search( + r'(?s)]+(?:poster="([^"]+)")?[^>]*>(.*)', webpage) if matches is None: raise ExtractorError('Unable to extract the video') @@ -39,7 +42,10 @@ class FKTVIE(InfoExtractor): self.report_warning('unable to extract thumbnail') urls = re.findall(r']+src="([^"]+)"', sources) - formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] + formats = [{ + 'url': url, + 'format_id': determine_ext(url), + } for url in urls] return { 'id': episode, 'title': title, From 8de28761c41ca0451e3bbe4b9ee236d6651cca44 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 18:17:48 +0800 Subject: [PATCH 1421/1424] [fktv] Fix a regex --- youtube_dl/extractor/fktv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index 0c14834f9..d9fc9952d 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -33,7 +33,8 @@ class FKTVIE(InfoExtractor): title = clean_html(self._html_search_regex( '

([^<]+)

', webpage, 'title')) matches = re.search( - r'(?s)]+(?:poster="([^"]+)")?[^>]*>(.*)', webpage) + r'(?s)])+(?:poster="([^"]+)")?[^>]*>(.*)', + webpage) if matches is None: raise ExtractorError('Unable to extract the video') From 4866b72eb2bf5747cde4654e75b1c1be0d456456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 25 Sep 2015 21:58:45 +0200 Subject: [PATCH 1422/1424] [fktv] Don't redefine 'url' in list comprehension Detected with flake8. --- youtube_dl/extractor/fktv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index d9fc9952d..fb9739b49 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -44,9 +44,9 @@ class FKTVIE(InfoExtractor): urls = re.findall(r']+src="([^"]+)"', sources) formats = [{ - 'url': url, + 'url': furl, 'format_id': determine_ext(url), - } for url in urls] + } for furl in urls] return { 'id': episode, 'title': title, From 4c917d0314f0442fd17d6f8ec8e583252167512c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 25 Sep 2015 22:02:48 +0200 Subject: [PATCH 1423/1424] [README.md] Document the 'duration' field in the output template (#6929) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2ed751791..80152071d 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,7 @@ The `-o` option allows users to indicate a template for the output file names. T - `playlist`: The sequence will be replaced by the name or the id of the playlist that contains the video. - `playlist_index`: The sequence will be replaced by the index of the video in the playlist padded with leading zeros according to the total length of the playlist. - `format_id`: The sequence will be replaced by the format code specified by `--format`. + - `duration`: The sequence will be replaced by the length of the video in seconds. The current default template is `%(title)s-%(id)s.%(ext)s`. From 08bea4adde5d5beb5f2a5cc875216b4205b90d50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 25 Sep 2015 22:34:02 +0200 Subject: [PATCH 1424/1424] Also run tests with python 3.5 --- .travis.yml | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e78a2fa76..cc21fae8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.2" - "3.3" - "3.4" + - "3.5" sudo: false script: nosetests test --verbose notifications: diff --git a/tox.ini b/tox.ini index cd805fe8a..48504329f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py33,py34 +envlist = py26,py27,py33,py34,py35 [testenv] deps = nose