From 1e0f4c34608c7f5815e3833bf2bcf33cb50c9a7f Mon Sep 17 00:00:00 2001 From: lkho Date: Mon, 28 Sep 2020 03:21:31 +0800 Subject: [PATCH 1/6] [viu:ott] add language_flag_id query param --- youtube_dl/extractor/viu.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/viu.py b/youtube_dl/extractor/viu.py index 3bd37525b..3b27014a8 100644 --- a/youtube_dl/extractor/viu.py +++ b/youtube_dl/extractor/viu.py @@ -168,7 +168,7 @@ class ViuPlaylistIE(ViuBaseIE): class ViuOTTIE(InfoExtractor): IE_NAME = 'viu:ott' - _VALID_URL = r'https?://(?:www\.)?viu\.com/ott/(?P[a-z]{2})/[a-z]{2}-[a-z]{2}/vod/(?P\d+)' + _VALID_URL = r'https?://(?:www\.)?viu\.com/ott/(?P[a-z]{2})/(?P[a-z]{2}-[a-z]{2})/vod/(?P\d+)' _TESTS = [{ 'url': 'http://www.viu.com/ott/sg/en-us/vod/3421/The%20Prime%20Minister%20and%20I', 'info_dict': { @@ -201,9 +201,13 @@ class ViuOTTIE(InfoExtractor): 'TH': 4, 'PH': 5, } + _LANGUAGE_FLAG = { + 'zh-hk': 1, + 'en-us': 3, + } def _real_extract(self, url): - country_code, video_id = re.match(self._VALID_URL, url).groups() + country_code, lang_code, video_id = re.match(self._VALID_URL, url).groups() query = { 'r': 'vod/ajax-detail', @@ -227,6 +231,7 @@ class ViuOTTIE(InfoExtractor): 'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code, video_id, 'Downloading stream info', query={ 'ccs_product_id': video_data['ccs_product_id'], + 'language_flag_id': self._LANGUAGE_FLAG.get(lang_code.lower()) or '3', }, headers={ 'Referer': url, 'Origin': re.search(r'https?://[^/]+', url).group(0), From 3253c8e877d93a685cc96fc010b1526999532b1e Mon Sep 17 00:00:00 2001 From: lkho Date: Sat, 10 Oct 2020 22:56:23 +0800 Subject: [PATCH 2/6] [viu:ott] handle preview duration limit param --- youtube_dl/extractor/viu.py | 41 +++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/viu.py b/youtube_dl/extractor/viu.py index 3b27014a8..b6d0237da 100644 --- a/youtube_dl/extractor/viu.py +++ b/youtube_dl/extractor/viu.py @@ -7,6 +7,7 @@ from .common import InfoExtractor from ..compat import ( compat_kwargs, compat_str, + compat_urlparse, ) from ..utils import ( ExtractorError, @@ -227,21 +228,43 @@ class ViuOTTIE(InfoExtractor): if not video_data: raise ExtractorError('This video is not available in your region.', expected=True) - stream_data = self._download_json( - 'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code, - video_id, 'Downloading stream info', query={ - 'ccs_product_id': video_data['ccs_product_id'], - 'language_flag_id': self._LANGUAGE_FLAG.get(lang_code.lower()) or '3', - }, headers={ - 'Referer': url, - 'Origin': re.search(r'https?://[^/]+', url).group(0), - })['data']['stream'] + duration_limit = False + query = { + 'ccs_product_id': video_data['ccs_product_id'], + 'language_flag_id': self._LANGUAGE_FLAG.get(lang_code.lower()) or '3', + } + headers = { + 'Referer': re.search(r'https?://[^/]+', url).group(0), + 'Origin': re.search(r'https?://[^/]+', url).group(0), + } + try: + stream_data = self._download_json( + 'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code, + video_id, 'Downloading stream info', query=query, headers=headers) + stream_data = stream_data['data']['stream'] + except KeyError: + # preview is limited to 3min for non-members + # retry with the duration limit set + duration_limit = True + query['duration'] = '180' + stream_data = self._download_json( + 'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code, + video_id, 'Downloading stream info', query=query, headers=headers) + stream_data = stream_data['data']['stream'] stream_sizes = stream_data.get('size', {}) formats = [] for vid_format, stream_url in stream_data.get('url', {}).items(): height = int_or_none(self._search_regex( r's(\d+)p', vid_format, 'height', default=None)) + + # by pass preview duration limit + if duration_limit: + temp = compat_urlparse.urlparse(stream_url) + query = dict(compat_urlparse.parse_qsl(temp.query, keep_blank_values=True)) + query.update({'duration': '9999999', 'duration_start': '0'}) + stream_url = temp._replace(query=compat_urlparse.urlencode(query)).geturl() + formats.append({ 'format_id': vid_format, 'url': stream_url, From 438bd67f43b7fe3b715c94ac4d1ef8a25c81d600 Mon Sep 17 00:00:00 2001 From: lkho Date: Mon, 12 Oct 2020 00:21:11 +0800 Subject: [PATCH 3/6] [viu:ott] support premium account login --- youtube_dl/extractor/viu.py | 78 ++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/viu.py b/youtube_dl/extractor/viu.py index b6d0237da..2bea6b719 100644 --- a/youtube_dl/extractor/viu.py +++ b/youtube_dl/extractor/viu.py @@ -8,11 +8,13 @@ from ..compat import ( compat_kwargs, compat_str, compat_urlparse, + compat_urllib_request, ) from ..utils import ( ExtractorError, int_or_none, ) +import json class ViuBaseIE(InfoExtractor): @@ -169,6 +171,7 @@ class ViuPlaylistIE(ViuBaseIE): class ViuOTTIE(InfoExtractor): IE_NAME = 'viu:ott' + _NETRC_MACHINE = 'viu' _VALID_URL = r'https?://(?:www\.)?viu\.com/ott/(?P[a-z]{2})/(?P[a-z]{2}-[a-z]{2})/vod/(?P\d+)' _TESTS = [{ 'url': 'http://www.viu.com/ott/sg/en-us/vod/3421/The%20Prime%20Minister%20and%20I', @@ -204,8 +207,46 @@ class ViuOTTIE(InfoExtractor): } _LANGUAGE_FLAG = { 'zh-hk': 1, + 'zh-cn': 2, 'en-us': 3, } + _user_info = None + + def _detect_error(self, response): + code = response.get('status', {}).get('code') + if code > 0: + message = response.get('status', {}).get('message') + raise ExtractorError('%s said: %s (%s)' % ( + self.IE_NAME, message, code), expected=True) + return response['data'] + + def _raise_login_required(self): + raise ExtractorError( + 'This video requires login. ' + 'Specify --username and --password or --netrc (machine: %s) ' + 'to provide account credentials.' % self._NETRC_MACHINE, + expected=True) + + def _login(self, country_code, video_id): + if not self._user_info: + username, password = self._get_login_info() + if username is None or password is None: + return + + data = self._download_json( + compat_urllib_request.Request( + 'https://www.viu.com/ott/%s/index.php' % country_code, method='POST'), + video_id, 'Logging in', errnote=False, fatal=False, + query={'r': 'user/login'}, + data=json.dumps({ + 'username': username, + 'password': password, + 'platform_flag_label': 'web', + }).encode()) + data = self._detect_error(data) + self._user_info = data['user'] + + return self._user_info def _real_extract(self, url): country_code, lang_code, video_id = re.match(self._VALID_URL, url).groups() @@ -241,16 +282,33 @@ class ViuOTTIE(InfoExtractor): stream_data = self._download_json( 'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code, video_id, 'Downloading stream info', query=query, headers=headers) - stream_data = stream_data['data']['stream'] - except KeyError: - # preview is limited to 3min for non-members - # retry with the duration limit set - duration_limit = True - query['duration'] = '180' - stream_data = self._download_json( - 'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code, - video_id, 'Downloading stream info', query=query, headers=headers) - stream_data = stream_data['data']['stream'] + stream_data = self._detect_error(stream_data)['stream'] + except (ExtractorError, KeyError): + stream_data = None + if video_data.get('user_level', 0) > 0: + user = self._login(country_code, video_id) + if user: + query['identity'] = user['identity'] + stream_data = self._download_json( + 'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code, + video_id, 'Downloading stream info', query=query, headers=headers) + stream_data = self._detect_error(stream_data).get('stream') + else: + # preview is limited to 3min for non-members + # try to bypass the duration limit + duration_limit = True + query['duration'] = '180' + stream_data = self._download_json( + 'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code, + video_id, 'Downloading stream info', query=query, headers=headers) + try: + stream_data = self._detect_error(stream_data)['stream'] + except (ExtractorError, KeyError): + # if still not working, give up + self._raise_login_required() + + if not stream_data: + raise ExtractorError('Cannot get stream info', expected=True) stream_sizes = stream_data.get('size', {}) formats = [] From 378b1c46356629fee91608328f3dafe57516227f Mon Sep 17 00:00:00 2001 From: lkho Date: Mon, 12 Oct 2020 00:37:30 +0800 Subject: [PATCH 4/6] [viu:ott] substitute duration from api data --- youtube_dl/extractor/viu.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/viu.py b/youtube_dl/extractor/viu.py index 2bea6b719..b809d3d00 100644 --- a/youtube_dl/extractor/viu.py +++ b/youtube_dl/extractor/viu.py @@ -320,7 +320,10 @@ class ViuOTTIE(InfoExtractor): if duration_limit: temp = compat_urlparse.urlparse(stream_url) query = dict(compat_urlparse.parse_qsl(temp.query, keep_blank_values=True)) - query.update({'duration': '9999999', 'duration_start': '0'}) + query.update({ + 'duration': video_data.get('time_duration', '9999999'), + 'duration_start': '0' + }) stream_url = temp._replace(query=compat_urlparse.urlencode(query)).geturl() formats.append({ From 24a3b815f9292ccf8da2e4632fc95286a286f255 Mon Sep 17 00:00:00 2001 From: lkho Date: Sat, 6 Jan 2018 22:22:22 +0800 Subject: [PATCH 5/6] [viu:ott] download entire series if --no-playlist is not specified --- youtube_dl/extractor/viu.py | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/youtube_dl/extractor/viu.py b/youtube_dl/extractor/viu.py index b809d3d00..68c36cf4b 100644 --- a/youtube_dl/extractor/viu.py +++ b/youtube_dl/extractor/viu.py @@ -13,6 +13,8 @@ from ..compat import ( from ..utils import ( ExtractorError, int_or_none, + smuggle_url, + unsmuggle_url, ) import json @@ -183,6 +185,7 @@ class ViuOTTIE(InfoExtractor): }, 'params': { 'skip_download': 'm3u8 download', + 'noplaylist': True, }, 'skip': 'Geo-restricted to Singapore', }, { @@ -195,6 +198,19 @@ class ViuOTTIE(InfoExtractor): }, 'params': { 'skip_download': 'm3u8 download', + 'noplaylist': True, + }, + 'skip': 'Geo-restricted to Hong Kong', + }, { + 'url': 'https://www.viu.com/ott/hk/zh-hk/vod/68776/%E6%99%82%E5%B0%9A%E5%AA%BD%E5%92%AA', + 'playlist_count': 12, + 'info_dict': { + 'id': '3916', + 'title': '時尚媽咪', + }, + 'params': { + 'skip_download': 'm3u8 download', + 'noplaylist': False, }, 'skip': 'Geo-restricted to Hong Kong', }] @@ -249,6 +265,7 @@ class ViuOTTIE(InfoExtractor): return self._user_info def _real_extract(self, url): + url, idata = unsmuggle_url(url, {}) country_code, lang_code, video_id = re.match(self._VALID_URL, url).groups() query = { @@ -269,6 +286,37 @@ class ViuOTTIE(InfoExtractor): if not video_data: raise ExtractorError('This video is not available in your region.', expected=True) + # return entire series as playlist if not --no-playlist + if not (self._downloader.params.get('noplaylist') or idata.get('force_noplaylist')): + series = product_data.get('series', {}) + product = series.get('product') + if product: + entries = [] + for entry in sorted(product, key=lambda x: int_or_none(x.get('number', 0))): + item_id = entry.get('product_id') + if not item_id: + continue + item_id = compat_str(item_id) + entries.append(self.url_result( + smuggle_url( + 'http://www.viu.com/ott/%s/%s/vod/%s/' % (country_code, lang_code, item_id), + {'force_noplaylist': True}), # prevent infinite recursion + 'ViuOTT', + item_id, + entry.get('synopsis', '').strip())) + + return self.playlist_result( + entries, + video_data.get('series_id'), + series.get('name'), + series.get('description')) + # else fall-through + + if self._downloader.params.get('noplaylist'): + self.to_screen( + 'Downloading only video %s in series %s because of --no-playlist' % + (video_id, video_data.get('series_id'))) + duration_limit = False query = { 'ccs_product_id': video_data['ccs_product_id'], From 9960d740397f67ee85b8ebb8d5c19114d4a529a1 Mon Sep 17 00:00:00 2001 From: lkho Date: Sun, 18 Oct 2020 23:01:36 +0800 Subject: [PATCH 6/6] [viu:ott] fix duration substitution --- youtube_dl/extractor/viu.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/viu.py b/youtube_dl/extractor/viu.py index b809d3d00..5e70ff128 100644 --- a/youtube_dl/extractor/viu.py +++ b/youtube_dl/extractor/viu.py @@ -320,9 +320,10 @@ class ViuOTTIE(InfoExtractor): if duration_limit: temp = compat_urlparse.urlparse(stream_url) query = dict(compat_urlparse.parse_qsl(temp.query, keep_blank_values=True)) + time_duration = int_or_none(video_data.get('time_duration')) query.update({ - 'duration': video_data.get('time_duration', '9999999'), - 'duration_start': '0' + 'duration': time_duration if time_duration > 0 else '9999999', + 'duration_start': '0', }) stream_url = temp._replace(query=compat_urlparse.urlencode(query)).geturl()