From 171a9ede1504edecf67cf7e0c477c046fcc3dcbf Mon Sep 17 00:00:00 2001 From: mrBliss Date: Tue, 31 Jan 2017 13:59:18 +0100 Subject: [PATCH 1/6] [vtm] Add extractor for member-only videos on VTM.be Implementation of the approach described in #9974. --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/vtm.py | 126 +++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 youtube_dl/extractor/vtm.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 24c478932..da8602577 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1174,6 +1174,7 @@ from .voxmedia import VoxMediaIE from .vporn import VpornIE from .vrt import VRTIE from .vrak import VrakIE +from .vtm import VTMIE from .vube import VubeIE from .vuclip import VuClipIE from .vvvvid import VVVVIDIE diff --git a/youtube_dl/extractor/vtm.py b/youtube_dl/extractor/vtm.py new file mode 100644 index 000000000..db42d72ef --- /dev/null +++ b/youtube_dl/extractor/vtm.py @@ -0,0 +1,126 @@ +from __future__ import unicode_literals + +import re + +from .generic import GenericIE +from .common import InfoExtractor +from ..utils import ( + urlencode_postdata, + compat_urllib_parse_urlencode, + ExtractorError, + remove_end, +) + + +class VTMIE(InfoExtractor): + """Download full episodes that require an account from vtm.be or q2.be. + + The generic extractor can be used to download clips that do no require an + account. + """ + _VALID_URL = r'https?://(?:www\.)?(?Pvtm|q2)\.be/video[/?].+?' + _NETRC_MACHINE = 'vtm' + _APIKEY = '3_HZ0FtkMW_gOyKlqQzW5_0FHRC7Nd5XpXJZcDdXY4pk5eES2ZWmejRW5egwVm4ug-' + _TESTS = [ + { + 'url': 'http://vtm.be/video/volledige-afleveringen/id/vtm_20170219_VM0678361_vtmwatch', + 'md5': '9357926318cdfcb10079d85d56f4e04e', + 'info_dict': { + 'id': 'vtm_20170219_VM0678361_vtmwatch', + 'ext': 'mp4', + 'title': 'Allemaal Chris afl. 6', + 'description': 'md5:4be86427521e7b07e0adb0c9c554ddb2', + }, + 'skip_download': True, + } + ] + + def _login(self): + (username, password) = self._get_login_info() + if username is None or password is None: + self.raise_login_required() + + auth_data = { + 'APIKey': self._APIKEY, + 'sdk': 'js_6.1', + 'format': 'json', + 'loginID': username, + 'password': password, + } + + auth_info = self._download_json( + 'https://accounts.eu1.gigya.com/accounts.login', None, + note='Logging in', errnote='Unable to log in', + data=urlencode_postdata(auth_data), fatal=True) + + error_message = auth_info.get('errorDetails') + if error_message: + raise ExtractorError( + 'Unable to login: %s' % error_message, expected=True) + + self._uid = auth_info['UID'] + self._uid_signature = auth_info['UIDSignature'] + self._signature_timestamp = auth_info['signatureTimestamp'] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + site_id = mobj.group('site_id') + + webpage = self._download_webpage(url, None) + + # There are two known types of URLs both pointing to the same video. + # The type of URL you get depends on how you navigate to the video. + # Unfortunately, only one type of URL contains the actual video id: + # + # http://vtm.be/video/volledige-afleveringen/id/257059160170000 + # + # The video id can also be something like: + # "vtm_20170213_VM0678492_vtmwatch" + # + # The other type of URL looks like: + # + # http://vtm.be/video?aid=166380 + # + # The 'aid' is not the video id we are looking for. There can also be + # a lot of very long parameters in addition to the 'aid' parameter. + # + # So to work in all cases, we extract the video_id from the webpage + # instead of the URL. + video_id = self._search_regex( + r'\\"vodId\\":\\"(.+?)\\"', webpage, 'video_id', default=None) + + # When no video_id is found, it was most likely a video not requiring + # authentication, so just fall back to the generic extractor + if not video_id: + return self.url_result(url, 'Generic') + + self._login() + + title = remove_end(self._og_search_title(webpage), ' - Volledige Afleveringen') + + description = self._html_search_regex( + r']+class="field-item\s+even">\s*

(.+?)

', + webpage, 'description', default=None) + + data_url = 'http://vod.medialaan.io/api/1.0/item/%s/video' % video_id + m3u8_data = { + 'app_id': 'vtm_watch' if site_id == 'vtm' else 'q2', + 'user_network': 'vtm-sso', + 'UID': self._uid, + 'UIDSignature': self._uid_signature, + 'signatureTimestamp': self._signature_timestamp, + } + data = self._download_json(data_url, video_id, query=m3u8_data) + + formats = self._extract_m3u8_formats( + data['response']['uri'], video_id, entry_protocol='m3u8_native', + ext='mp4', m3u8_id='hls') + + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'formats': formats, + } From 2f9a673a1af8a49044f0519a6ae52b24a888ac40 Mon Sep 17 00:00:00 2001 From: mrBliss Date: Sun, 12 Mar 2017 18:42:31 +0100 Subject: [PATCH 2/6] [vtm] More robust title extraction --- youtube_dl/extractor/vtm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vtm.py b/youtube_dl/extractor/vtm.py index db42d72ef..a3834ebc8 100644 --- a/youtube_dl/extractor/vtm.py +++ b/youtube_dl/extractor/vtm.py @@ -96,7 +96,8 @@ class VTMIE(InfoExtractor): self._login() - title = remove_end(self._og_search_title(webpage), ' - Volledige Afleveringen') + title = self._html_search_regex( + r'\\"title\\":\\"(.+?)\\"', webpage, 'title', default=None) description = self._html_search_regex( r']+class="field-item\s+even">\s*

(.+?)

', From ce089ad2840ae7861fde5a40e77e39fa31db2630 Mon Sep 17 00:00:00 2001 From: mrBliss Date: Sun, 12 Mar 2017 18:17:25 +0100 Subject: [PATCH 3/6] [vtm] Add tests for the different video ids --- youtube_dl/extractor/vtm.py | 43 ++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/youtube_dl/extractor/vtm.py b/youtube_dl/extractor/vtm.py index a3834ebc8..e692bf4fa 100644 --- a/youtube_dl/extractor/vtm.py +++ b/youtube_dl/extractor/vtm.py @@ -24,7 +24,6 @@ class VTMIE(InfoExtractor): _TESTS = [ { 'url': 'http://vtm.be/video/volledige-afleveringen/id/vtm_20170219_VM0678361_vtmwatch', - 'md5': '9357926318cdfcb10079d85d56f4e04e', 'info_dict': { 'id': 'vtm_20170219_VM0678361_vtmwatch', 'ext': 'mp4', @@ -32,7 +31,27 @@ class VTMIE(InfoExtractor): 'description': 'md5:4be86427521e7b07e0adb0c9c554ddb2', }, 'skip_download': True, - } + }, + { + 'url': 'http://vtm.be/video/volledige-afleveringen/id/257107153551000', + 'info_dict': { + 'id': '257107153551000', + 'ext': 'mp4', + 'title': 'Blind Getrouwd afl.6', + 'description': 'md5:9ed26e8486ad0c1ddade4b066d83100f', + }, + 'skip_download': True, + }, + { + 'url': 'http://vtm.be/video?aid=163157', + 'info_dict': { + 'id': '257068686507000', + 'ext': 'mp4', + 'title': 'Amigo\'s afl.8', + 'description': 'md5:1db8963594a1829839dabd55ef4f6a0b', + }, + 'skip_download': True, + }, ] def _login(self): @@ -68,24 +87,8 @@ class VTMIE(InfoExtractor): webpage = self._download_webpage(url, None) - # There are two known types of URLs both pointing to the same video. - # The type of URL you get depends on how you navigate to the video. - # Unfortunately, only one type of URL contains the actual video id: - # - # http://vtm.be/video/volledige-afleveringen/id/257059160170000 - # - # The video id can also be something like: - # "vtm_20170213_VM0678492_vtmwatch" - # - # The other type of URL looks like: - # - # http://vtm.be/video?aid=166380 - # - # The 'aid' is not the video id we are looking for. There can also be - # a lot of very long parameters in addition to the 'aid' parameter. - # - # So to work in all cases, we extract the video_id from the webpage - # instead of the URL. + # The URL sometimes contains the video id, but not always, e.g., test + # case 3. Fortunately, all webpages contain the video id. video_id = self._search_regex( r'\\"vodId\\":\\"(.+?)\\"', webpage, 'video_id', default=None) From 418866d78b2b98b953e2ba22a44df4a8a543a1ef Mon Sep 17 00:00:00 2001 From: mrBliss Date: Sun, 12 Mar 2017 19:15:20 +0100 Subject: [PATCH 4/6] [vtm] Add test for Q2 --- youtube_dl/extractor/vtm.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/youtube_dl/extractor/vtm.py b/youtube_dl/extractor/vtm.py index e692bf4fa..7aa06f878 100644 --- a/youtube_dl/extractor/vtm.py +++ b/youtube_dl/extractor/vtm.py @@ -52,6 +52,16 @@ class VTMIE(InfoExtractor): }, 'skip_download': True, }, + { + 'url': 'http://www.q2.be/video/volledige-afleveringen/id/2be_20170301_VM0684442_q2', + 'info_dict': { + 'id': '2be_20170301_VM0684442_q2', + 'ext': 'mp4', + 'title': 'Modern Family afl. 15', + 'description': 'md5:9866dd440480ee5ad5195382dd33d068', + }, + 'skip_download': True, + }, ] def _login(self): From 4c163143c3c388806b3e7cb824526f71dc13b7ec Mon Sep 17 00:00:00 2001 From: mrBliss Date: Mon, 13 Mar 2017 16:55:32 +0100 Subject: [PATCH 5/6] [vtm] Address feedback except for the only_matching comment --- youtube_dl/extractor/vtm.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/vtm.py b/youtube_dl/extractor/vtm.py index 7aa06f878..8f83b7363 100644 --- a/youtube_dl/extractor/vtm.py +++ b/youtube_dl/extractor/vtm.py @@ -64,6 +64,9 @@ class VTMIE(InfoExtractor): }, ] + def _real_initialize(self): + self._logged_in = False + def _login(self): (username, password) = self._get_login_info() if username is None or password is None: @@ -91,23 +94,26 @@ class VTMIE(InfoExtractor): self._uid_signature = auth_info['UIDSignature'] self._signature_timestamp = auth_info['signatureTimestamp'] + self._logged_in = True + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) site_id = mobj.group('site_id') - webpage = self._download_webpage(url, None) + webpage = self._download_webpage(url, None, "Downloading webpage") # The URL sometimes contains the video id, but not always, e.g., test - # case 3. Fortunately, all webpages contain the video id. + # case 3. Fortunately, all webpages of videos requiring authentication + # contain the video id. video_id = self._search_regex( r'\\"vodId\\":\\"(.+?)\\"', webpage, 'video_id', default=None) - # When no video_id is found, it was most likely a video not requiring - # authentication, so just fall back to the generic extractor + # It was most likely a video not requiring authentication. if not video_id: return self.url_result(url, 'Generic') - self._login() + if not self._logged_in: + self._login() title = self._html_search_regex( r'\\"title\\":\\"(.+?)\\"', webpage, 'title', default=None) From 9cd898a8d5062e07fcb2d43a4ddfbb925f56df2e Mon Sep 17 00:00:00 2001 From: mrBliss Date: Mon, 13 Mar 2017 17:34:02 +0100 Subject: [PATCH 6/6] [vtm] Reorganise tests Make all tests that require authentication except the first `only_matching`. Add a test for a video that does not require authentication. --- youtube_dl/extractor/vtm.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/youtube_dl/extractor/vtm.py b/youtube_dl/extractor/vtm.py index 8f83b7363..f0a70040b 100644 --- a/youtube_dl/extractor/vtm.py +++ b/youtube_dl/extractor/vtm.py @@ -34,33 +34,23 @@ class VTMIE(InfoExtractor): }, { 'url': 'http://vtm.be/video/volledige-afleveringen/id/257107153551000', - 'info_dict': { - 'id': '257107153551000', - 'ext': 'mp4', - 'title': 'Blind Getrouwd afl.6', - 'description': 'md5:9ed26e8486ad0c1ddade4b066d83100f', - }, - 'skip_download': True, + 'only_matching': True, }, { 'url': 'http://vtm.be/video?aid=163157', - 'info_dict': { - 'id': '257068686507000', - 'ext': 'mp4', - 'title': 'Amigo\'s afl.8', - 'description': 'md5:1db8963594a1829839dabd55ef4f6a0b', - }, - 'skip_download': True, + 'only_matching': True, }, { 'url': 'http://www.q2.be/video/volledige-afleveringen/id/2be_20170301_VM0684442_q2', + 'only_matching': True, + }, + { + 'url': 'http://vtm.be/video?aid=168332', 'info_dict': { - 'id': '2be_20170301_VM0684442_q2', + 'id': 'video?aid=168332', 'ext': 'mp4', - 'title': 'Modern Family afl. 15', - 'description': 'md5:9866dd440480ee5ad5195382dd33d068', + 'title': 'Videozone', }, - 'skip_download': True, }, ]