From 89fd03079b83e1dfae016e4e28746b00befe8b2d Mon Sep 17 00:00:00 2001 From: mrBliss Date: Mon, 15 May 2017 16:46:55 +0200 Subject: [PATCH 1/6] [vier] Improve extraction + Add support for authentication * Bypass authentication when no credentials provded * Improve extraction robustness --- youtube_dl/extractor/vier.py | 74 +++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py index 5ef7635b6..b6efaa284 100644 --- a/youtube_dl/extractor/vier.py +++ b/youtube_dl/extractor/vier.py @@ -5,12 +5,17 @@ import re import itertools from .common import InfoExtractor +from ..utils import ( + ExtractorError, + urlencode_postdata, +) class VierIE(InfoExtractor): IE_NAME = 'vier' IE_DESC = 'vier.be and vijf.be' _VALID_URL = r'https?://(?:www\.)?(?Pvier|vijf)\.be/(?:[^/]+/videos/(?P[^/]+)(?:/(?P\d+))?|video/v3/embed/(?P\d+))' + _NETRC_MACHINE = 'vier' _TESTS = [{ 'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129', 'info_dict': { @@ -24,19 +29,47 @@ class VierIE(InfoExtractor): # m3u8 download 'skip_download': True, }, + # 'skip': 'Requires account credentials', }, { 'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614', 'info_dict': { 'id': '2561614', 'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas', 'ext': 'mp4', - 'title': 'ZO grappig: Temptation Island hosts moeten kiezen tussen onmogelijke dilemma\'s', + 'title': 'EXTRA: Temptation Island hosts moeten kiezen tussen onmogelijke dilemma\'s', 'description': 'Het spel is simpel: Annelien Coorevits en Rick Brandsteder krijgen telkens 2 dilemma\'s voorgeschoteld en ze MOETEN een keuze maken.', }, 'params': { # m3u8 download 'skip_download': True, }, + }, { + 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839', + 'info_dict': { + 'id': '2674839', + 'display_id': 'jani-gaat-naar-tokio-aflevering-4', + 'ext': 'mp4', + 'title': 'Jani gaat naar Tokio - Aflevering 4', + 'description': 'Bekijk hier de volledige vierde aflevering van het 2de seizoen van Jani gaat...', + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + 'skip': 'Requires account credentials', + }, { + 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839', + 'info_dict': { + 'id': '2674839', + 'display_id': 'jani-gaat-naar-tokio-aflevering-4', + 'ext': 'mp4', + 'title': 'jani-gaat-naar-tokio-aflevering-4', + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + 'expected_warnings': ['Log in to extract metadata'], }, { 'url': 'http://www.vier.be/planb/videos/mieren-herders-van-de-bladluizen', 'only_matching': True, @@ -45,17 +78,54 @@ class VierIE(InfoExtractor): 'only_matching': True, }] + def _real_initialize(self): + self._logged_in = False + + def _login(self, site): + username, password = self._get_login_info() + if username is None or password is None: + return + + login_page = self._download_webpage( + 'http://www.%s.be/user/login' % site, + None, note='Logging in', errnote='Unable to log in', + data=urlencode_postdata({ + 'form_id': 'user_login', + 'name': username, + 'pass': password, + }), + headers={'Content-Type': 'application/x-www-form-urlencoded'}) + + login_error = self._html_search_regex( + r'(?s)
\s*
\s*(.+?)<', + login_page, 'login error', default=None) + if login_error: + self.report_warning('Unable to log in: %s' % login_error) + else: + self._logged_in = True + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) embed_id = mobj.group('embed_id') display_id = mobj.group('display_id') or embed_id + video_id = mobj.group('id') or embed_id site = mobj.group('site') + if not self._logged_in: + self._login(site) + webpage = self._download_webpage(url, display_id) + if r'id="user-login"' in webpage: + self.report_warning( + 'Log in to extract metadata', video_id=display_id) + webpage = self._download_webpage( + 'http://www.%s.be/video/v3/embed/%s' % (site, video_id), + display_id) + video_id = self._search_regex( [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'], - webpage, 'video id') + webpage, 'video id', default=video_id) application = self._search_regex( [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'], webpage, 'application', default=site + '_vod') From 7073015a23f96ce8ca0400286051d53cb9237a9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 15 May 2017 22:00:53 +0700 Subject: [PATCH 2/6] [vier] PEP 8 and cleanup --- youtube_dl/extractor/vier.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py index b6efaa284..a98bd5c81 100644 --- a/youtube_dl/extractor/vier.py +++ b/youtube_dl/extractor/vier.py @@ -5,10 +5,7 @@ import re import itertools from .common import InfoExtractor -from ..utils import ( - ExtractorError, - urlencode_postdata, -) +from ..utils import urlencode_postdata class VierIE(InfoExtractor): @@ -18,6 +15,7 @@ class VierIE(InfoExtractor): _NETRC_MACHINE = 'vier' _TESTS = [{ 'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129', + 'md5': 'e4ae2054a6b040ef1e289e20d111b46e', 'info_dict': { 'id': '16129', 'display_id': 'het-wordt-warm-de-moestuin', @@ -25,22 +23,16 @@ class VierIE(InfoExtractor): 'title': 'Het wordt warm in De Moestuin', 'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...', }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - # 'skip': 'Requires account credentials', }, { 'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614', 'info_dict': { 'id': '2561614', 'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas', 'ext': 'mp4', - 'title': 'EXTRA: Temptation Island hosts moeten kiezen tussen onmogelijke dilemma\'s', - 'description': 'Het spel is simpel: Annelien Coorevits en Rick Brandsteder krijgen telkens 2 dilemma\'s voorgeschoteld en ze MOETEN een keuze maken.', + 'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7', + 'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe', }, 'params': { - # m3u8 download 'skip_download': True, }, }, { @@ -50,14 +42,15 @@ class VierIE(InfoExtractor): 'display_id': 'jani-gaat-naar-tokio-aflevering-4', 'ext': 'mp4', 'title': 'Jani gaat naar Tokio - Aflevering 4', - 'description': 'Bekijk hier de volledige vierde aflevering van het 2de seizoen van Jani gaat...', + 'description': 'md5:2d169e8186ae4247e50c99aaef97f7b2', }, 'params': { - # m3u8 download 'skip_download': True, }, 'skip': 'Requires account credentials', }, { + # Requires account credentials but bypassed extraction via v3/embed page + # without metadata 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839', 'info_dict': { 'id': '2674839', @@ -66,12 +59,12 @@ class VierIE(InfoExtractor): 'title': 'jani-gaat-naar-tokio-aflevering-4', }, 'params': { - # m3u8 download 'skip_download': True, }, 'expected_warnings': ['Log in to extract metadata'], }, { - 'url': 'http://www.vier.be/planb/videos/mieren-herders-van-de-bladluizen', + # Without video id in URL + 'url': 'http://www.vier.be/planb/videos/dit-najaar-plan-b', 'only_matching': True, }, { 'url': 'http://www.vier.be/video/v3/embed/16129', @@ -125,7 +118,7 @@ class VierIE(InfoExtractor): video_id = self._search_regex( [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'], - webpage, 'video id', default=video_id) + webpage, 'video id', default=video_id or display_id) application = self._search_regex( [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'], webpage, 'application', default=site + '_vod') From 6f766798049418c15e3c1b62c046e507093993dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 16 May 2017 22:11:34 +0700 Subject: [PATCH 3/6] [extractor/common] Add support for schemeless URLs in _extract_wowza_formats (closes #13088, closes #13092) --- youtube_dl/extractor/common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 0bbb1103b..74b6f1197 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -2205,8 +2205,9 @@ class InfoExtractor(object): def _extract_wowza_formats(self, url, video_id, m3u8_entry_protocol='m3u8_native', skip_protocols=[]): url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url) - url_base = self._search_regex(r'(?:https?|rtmp|rtsp)(://[^?]+)', url, 'format url') - http_base_url = 'http' + url_base + url_base = self._search_regex( + r'(?:(?:https?|rtmp|rtsp):)?(//[^?]+)', url, 'format url') + http_base_url = self._proto_relative_url(url_base, scheme='http:') formats = [] if 'm3u8' not in skip_protocols: formats.extend(self._extract_m3u8_formats( From f2e2f0c777dc9e541d89b742ac12e7ad37d8cb8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 17 May 2017 22:19:33 +0700 Subject: [PATCH 4/6] [extractor/common] Fix rtmp and rtsp formats' URLs in _extract_wowza_formats --- youtube_dl/extractor/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 74b6f1197..fec39da8b 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -2207,7 +2207,7 @@ class InfoExtractor(object): url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url) url_base = self._search_regex( r'(?:(?:https?|rtmp|rtsp):)?(//[^?]+)', url, 'format url') - http_base_url = self._proto_relative_url(url_base, scheme='http:') + http_base_url = '%s:%s' % ('http', url_base) formats = [] if 'm3u8' not in skip_protocols: formats.extend(self._extract_m3u8_formats( @@ -2241,7 +2241,7 @@ class InfoExtractor(object): for protocol in ('rtmp', 'rtsp'): if protocol not in skip_protocols: formats.append({ - 'url': protocol + url_base, + 'url': '%s:%s' % (protocol, url_base), 'format_id': protocol, 'protocol': protocol, }) From 36b226d48fd0b366dcf99a646550181639301eb2 Mon Sep 17 00:00:00 2001 From: mrBliss Date: Fri, 24 Mar 2017 10:34:35 +0100 Subject: [PATCH 5/6] [vier] Extract more info Extract the `episode_number` and `upload_date`. Also extract the real `description`. --- youtube_dl/extractor/vier.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py index a98bd5c81..7fb718343 100644 --- a/youtube_dl/extractor/vier.py +++ b/youtube_dl/extractor/vier.py @@ -5,7 +5,11 @@ import re import itertools from .common import InfoExtractor -from ..utils import urlencode_postdata +from ..utils import ( + urlencode_postdata, + int_or_none, + unified_strdate, +) class VierIE(InfoExtractor): @@ -22,6 +26,7 @@ class VierIE(InfoExtractor): 'ext': 'mp4', 'title': 'Het wordt warm in De Moestuin', 'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...', + 'upload_date': '20121025', }, }, { 'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614', @@ -31,6 +36,7 @@ class VierIE(InfoExtractor): 'ext': 'mp4', 'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7', 'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe', + 'upload_date': '20170228', }, 'params': { 'skip_download': True, @@ -42,7 +48,9 @@ class VierIE(InfoExtractor): 'display_id': 'jani-gaat-naar-tokio-aflevering-4', 'ext': 'mp4', 'title': 'Jani gaat naar Tokio - Aflevering 4', - 'description': 'md5:2d169e8186ae4247e50c99aaef97f7b2', + 'description': 'md5:aa8d611541db6ae9e863125704511f88', + 'upload_date': '20170501', + 'episode_number': 4, }, 'params': { 'skip_download': True, @@ -131,14 +139,28 @@ class VierIE(InfoExtractor): self._sort_formats(formats) title = self._og_search_title(webpage, default=display_id) - description = self._og_search_description(webpage, default=None) thumbnail = self._og_search_thumbnail(webpage, default=None) + description = self._html_search_regex( + r'''(?x)\s* + (?:\s*)* +

\s*(?:)?(.+?)\s* + (?:\s*)* + (\d{2}/\d{2}/\d{4})''', + webpage, 'upload_date', default=None)) return { 'id': video_id, 'display_id': display_id, 'title': title, 'description': description, + 'episode_number': episode_number, + 'upload_date': upload_date, 'thumbnail': thumbnail, 'formats': formats, } From 650bd9471697390a9cdef2a54d35ec6dd5dc49ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 17 May 2017 23:38:27 +0700 Subject: [PATCH 6/6] [vier] Relax regexes and extract more metadata (closes #12539) --- youtube_dl/extractor/vier.py | 39 ++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py index 7fb718343..3e67eb8c2 100644 --- a/youtube_dl/extractor/vier.py +++ b/youtube_dl/extractor/vier.py @@ -27,6 +27,8 @@ class VierIE(InfoExtractor): 'title': 'Het wordt warm in De Moestuin', 'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...', 'upload_date': '20121025', + 'series': 'Plan B', + 'tags': ['De Moestuin', 'Moestuin', 'meisjes', 'Tomaat', 'Wim', 'Droom'], }, }, { 'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614', @@ -37,6 +39,8 @@ class VierIE(InfoExtractor): 'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7', 'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe', 'upload_date': '20170228', + 'series': 'Temptation Island', + 'tags': list, }, 'params': { 'skip_download': True, @@ -50,7 +54,9 @@ class VierIE(InfoExtractor): 'title': 'Jani gaat naar Tokio - Aflevering 4', 'description': 'md5:aa8d611541db6ae9e863125704511f88', 'upload_date': '20170501', + 'series': 'Jani gaat', 'episode_number': 4, + 'tags': ['Jani Gaat', 'Volledige Aflevering'], }, 'params': { 'skip_download': True, @@ -135,33 +141,36 @@ class VierIE(InfoExtractor): webpage, 'filename') playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename) - formats = self._extract_wowza_formats(playlist_url, display_id, skip_protocols=['dash']) + formats = self._extract_wowza_formats( + playlist_url, display_id, skip_protocols=['dash']) self._sort_formats(formats) title = self._og_search_title(webpage, default=display_id) - thumbnail = self._og_search_thumbnail(webpage, default=None) description = self._html_search_regex( - r'''(?x)\s* - (?:\s*)* -

\s*(?:)?(.+?)]+\bclass=(["\'])[^>]*?\bfield-type-text-with-summary\b[^>]*?\1[^>]*>.*?

(?P.+?)

', + webpage, 'description', default=None, group='value') + thumbnail = self._og_search_thumbnail(webpage, default=None) upload_date = unified_strdate(self._html_search_regex( - r'''(?x)\s* - (?:\s*)* - (\d{2}/\d{2}/\d{4})''', - webpage, 'upload_date', default=None)) + r'(?s)]+\bclass=(["\'])[^>]*?\bfield-name-post-date\b[^>]*?\1[^>]*>.*?(?P\d{2}/\d{2}/\d{4})', + webpage, 'upload date', default=None, group='value')) + + series = self._search_regex( + r'data-program=(["\'])(?P(?:(?!\1).)+)\1', webpage, + 'series', default=None, group='value') + episode_number = int_or_none(self._search_regex( + r'(?i)aflevering (\d+)', title, 'episode number', default=None)) + tags = re.findall(r']+\bhref=["\']/tags/[^>]+>([^<]+)<', webpage) return { 'id': video_id, 'display_id': display_id, 'title': title, 'description': description, - 'episode_number': episode_number, - 'upload_date': upload_date, 'thumbnail': thumbnail, + 'upload_date': upload_date, + 'series': series, + 'episode_number': episode_number, + 'tags': tags, 'formats': formats, }