From 30d2664c593f2c1ca823ba9edfa15e832b7bb66a Mon Sep 17 00:00:00 2001 From: Kyle Date: Sat, 8 Jun 2019 09:11:22 +0900 Subject: [PATCH 1/4] Add playlist support to BrightcoveNewIE. --- youtube_dl/extractor/brightcove.py | 138 ++++++++++++++++------------- 1 file changed, 77 insertions(+), 61 deletions(-) diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index c0345e2c3..33c204456 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -32,6 +32,7 @@ from ..utils import ( update_url_query, clean_html, mimetype2ext, + url_or_none, ) @@ -483,7 +484,7 @@ class BrightcoveLegacyIE(InfoExtractor): class BrightcoveNewIE(AdobePassIE): IE_NAME = 'brightcove:new' - _VALID_URL = r'https?://players\.brightcove\.net/(?P\d+)/(?P[^/]+)_(?P[^/]+)/index\.html\?.*videoId=(?P\d+|ref:[^&]+)' + _VALID_URL = r'https?://players\.brightcove\.net/(?P\d+)/(?P[^/]+)_(?P[^/]+)/index\.html\?.*(?Pvideo|playlist)Id=(?P\d+|ref:[^&]+)' _TESTS = [{ 'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001', 'md5': 'c8100925723840d4b0d243f7025703be', @@ -516,6 +517,19 @@ class BrightcoveNewIE(AdobePassIE): # m3u8 download 'skip_download': True, } + }, { + # playlist streams + 'url': 'http://players.brightcove.net/5690807595001/HyZNerRl7_default/index.html?playlistId=5743160747001', + 'info_dict': { + 'id': '5743160747001', + 'ext': 'mp4', + 'title': 'prod ntv', + 'uploader_id': '5690807595001', + }, + 'params': { + # m3u8 download + 'skip_download': True, + } }, { # ref: prefixed video id 'url': 'http://players.brightcove.net/3910869709001/21519b5c-4b3b-4363-accb-bdc8f358f823_default/index.html?videoId=ref:7069442', @@ -600,68 +614,70 @@ class BrightcoveNewIE(AdobePassIE): title = json_data['name'].strip() formats = [] - for source in json_data.get('sources', []): - container = source.get('container') - ext = mimetype2ext(source.get('type')) - src = source.get('src') - # https://support.brightcove.com/playback-api-video-fields-reference#key_systems_object - if ext == 'ism' or container == 'WVM' or source.get('key_systems'): - continue - elif ext == 'm3u8' or container == 'M2TS': - if not src: + # playlistId data starts from `videos`, videoId from `sources` + for vid in json_data.get('videos', [json_data]): + for source in vid.get('sources', []): + container = source.get('container') + ext = mimetype2ext(source.get('type')) + src = source.get('src') + # https://support.brightcove.com/playback-api-video-fields-reference#key_systems_object + if ext == 'ism' or container == 'WVM' or source.get('key_systems'): continue - formats.extend(self._extract_m3u8_formats( - src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)) - elif ext == 'mpd': - if not src: - continue - formats.extend(self._extract_mpd_formats(src, video_id, 'dash', fatal=False)) - else: - streaming_src = source.get('streaming_src') - stream_name, app_name = source.get('stream_name'), source.get('app_name') - if not src and not streaming_src and (not stream_name or not app_name): - continue - tbr = float_or_none(source.get('avg_bitrate'), 1000) - height = int_or_none(source.get('height')) - width = int_or_none(source.get('width')) - f = { - 'tbr': tbr, - 'filesize': int_or_none(source.get('size')), - 'container': container, - 'ext': ext or container.lower(), - } - if width == 0 and height == 0: - f.update({ - 'vcodec': 'none', - }) + elif ext == 'm3u8' or container == 'M2TS': + if not src: + continue + formats.extend(self._extract_m3u8_formats( + src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)) + elif ext == 'mpd': + if not src: + continue + formats.extend(self._extract_mpd_formats(src, video_id, 'dash', fatal=False)) else: - f.update({ - 'width': width, - 'height': height, - 'vcodec': source.get('codec'), - }) + streaming_src = source.get('streaming_src') + stream_name, app_name = source.get('stream_name'), source.get('app_name') + if not src and not streaming_src and (not stream_name or not app_name): + continue + tbr = float_or_none(source.get('avg_bitrate'), 1000) + height = int_or_none(source.get('height')) + width = int_or_none(source.get('width')) + f = { + 'tbr': tbr, + 'filesize': int_or_none(source.get('size')), + 'container': container, + 'ext': ext or container.lower(), + } + if width == 0 and height == 0: + f.update({ + 'vcodec': 'none', + }) + else: + f.update({ + 'width': width, + 'height': height, + 'vcodec': source.get('codec'), + }) - def build_format_id(kind): - format_id = kind - if tbr: - format_id += '-%dk' % int(tbr) - if height: - format_id += '-%dp' % height - return format_id + def build_format_id(kind): + format_id = kind + if tbr: + format_id += '-%dk' % int(tbr) + if height: + format_id += '-%dp' % height + return format_id - if src or streaming_src: - f.update({ - 'url': src or streaming_src, - 'format_id': build_format_id('http' if src else 'http-streaming'), - 'source_preference': 0 if src else -1, - }) - else: - f.update({ - 'url': app_name, - 'play_path': stream_name, - 'format_id': build_format_id('rtmp'), - }) - formats.append(f) + if src or streaming_src: + f.update({ + 'url': src or streaming_src, + 'format_id': build_format_id('http' if src else 'http-streaming'), + 'source_preference': 0 if src else -1, + }) + else: + f.update({ + 'url': app_name, + 'play_path': stream_name, + 'format_id': build_format_id('rtmp'), + }) + formats.append(f) if not formats: # for sonyliv.com DRM protected videos s3_source_url = json_data.get('custom_fields', {}).get('s3sourceurl') @@ -715,7 +731,7 @@ class BrightcoveNewIE(AdobePassIE): 'ip_blocks': smuggled_data.get('geo_ip_blocks'), }) - account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups() + account_id, player_id, embed, content_type, video_id = re.match(self._VALID_URL, url).groups() webpage = self._download_webpage( 'http://players.brightcove.net/%s/%s_%s/index.min.js' @@ -736,7 +752,7 @@ class BrightcoveNewIE(AdobePassIE): r'policyKey\s*:\s*(["\'])(?P.+?)\1', webpage, 'policy key', group='pk') - api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id) + api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/%ss/%s' % (account_id, content_type, video_id) headers = { 'Accept': 'application/json;pk=%s' % policy_key, } From 93748c89e14ab9f21897b0858bebc648803fb4c5 Mon Sep 17 00:00:00 2001 From: Kyle Date: Tue, 18 Jun 2019 09:13:47 +0900 Subject: [PATCH 2/4] Add playlist format extraction. --- youtube_dl/extractor/brightcove.py | 133 ++++++++++++++++------------- 1 file changed, 72 insertions(+), 61 deletions(-) diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index 33c204456..a0a11628f 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -32,7 +32,6 @@ from ..utils import ( update_url_query, clean_html, mimetype2ext, - url_or_none, ) @@ -614,70 +613,68 @@ class BrightcoveNewIE(AdobePassIE): title = json_data['name'].strip() formats = [] - # playlistId data starts from `videos`, videoId from `sources` - for vid in json_data.get('videos', [json_data]): - for source in vid.get('sources', []): - container = source.get('container') - ext = mimetype2ext(source.get('type')) - src = source.get('src') - # https://support.brightcove.com/playback-api-video-fields-reference#key_systems_object - if ext == 'ism' or container == 'WVM' or source.get('key_systems'): + for source in json_data.get('sources', []): + container = source.get('container') + ext = mimetype2ext(source.get('type')) + src = source.get('src') + # https://support.brightcove.com/playback-api-video-fields-reference#key_systems_object + if ext == 'ism' or container == 'WVM' or source.get('key_systems'): + continue + elif ext == 'm3u8' or container == 'M2TS': + if not src: continue - elif ext == 'm3u8' or container == 'M2TS': - if not src: - continue - formats.extend(self._extract_m3u8_formats( - src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)) - elif ext == 'mpd': - if not src: - continue - formats.extend(self._extract_mpd_formats(src, video_id, 'dash', fatal=False)) + formats.extend(self._extract_m3u8_formats( + src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)) + elif ext == 'mpd': + if not src: + continue + formats.extend(self._extract_mpd_formats(src, video_id, 'dash', fatal=False)) + else: + streaming_src = source.get('streaming_src') + stream_name, app_name = source.get('stream_name'), source.get('app_name') + if not src and not streaming_src and (not stream_name or not app_name): + continue + tbr = float_or_none(source.get('avg_bitrate'), 1000) + height = int_or_none(source.get('height')) + width = int_or_none(source.get('width')) + f = { + 'tbr': tbr, + 'filesize': int_or_none(source.get('size')), + 'container': container, + 'ext': ext or container.lower(), + } + if width == 0 and height == 0: + f.update({ + 'vcodec': 'none', + }) else: - streaming_src = source.get('streaming_src') - stream_name, app_name = source.get('stream_name'), source.get('app_name') - if not src and not streaming_src and (not stream_name or not app_name): - continue - tbr = float_or_none(source.get('avg_bitrate'), 1000) - height = int_or_none(source.get('height')) - width = int_or_none(source.get('width')) - f = { - 'tbr': tbr, - 'filesize': int_or_none(source.get('size')), - 'container': container, - 'ext': ext or container.lower(), - } - if width == 0 and height == 0: - f.update({ - 'vcodec': 'none', - }) - else: - f.update({ - 'width': width, - 'height': height, - 'vcodec': source.get('codec'), - }) + f.update({ + 'width': width, + 'height': height, + 'vcodec': source.get('codec'), + }) - def build_format_id(kind): - format_id = kind - if tbr: - format_id += '-%dk' % int(tbr) - if height: - format_id += '-%dp' % height - return format_id + def build_format_id(kind): + format_id = kind + if tbr: + format_id += '-%dk' % int(tbr) + if height: + format_id += '-%dp' % height + return format_id - if src or streaming_src: - f.update({ - 'url': src or streaming_src, - 'format_id': build_format_id('http' if src else 'http-streaming'), - 'source_preference': 0 if src else -1, - }) - else: - f.update({ - 'url': app_name, - 'play_path': stream_name, - 'format_id': build_format_id('rtmp'), - }) - formats.append(f) + if src or streaming_src: + f.update({ + 'url': src or streaming_src, + 'format_id': build_format_id('http' if src else 'http-streaming'), + 'source_preference': 0 if src else -1, + }) + else: + f.update({ + 'url': app_name, + 'play_path': stream_name, + 'format_id': build_format_id('rtmp'), + }) + formats.append(f) if not formats: # for sonyliv.com DRM protected videos s3_source_url = json_data.get('custom_fields', {}).get('s3sourceurl') @@ -724,6 +721,17 @@ class BrightcoveNewIE(AdobePassIE): 'is_live': is_live, } + def _extract_playlist(self, json_data, headers): + playlist_id = json_data.get('id') + playlist_title = json_data.get('name') + playlist_description = json_data.get('description') + playlist_vids = json_data.get('videos', []) + + return self.playlist_result( + [self._parse_brightcove_metadata(vid, vid.get('id'), headers) for vid in playlist_vids], + playlist_id, playlist_title, playlist_description, + ) + def _real_extract(self, url): url, smuggled_data = unsmuggle_url(url, {}) self._initialize_geo_bypass({ @@ -787,5 +795,8 @@ class BrightcoveNewIE(AdobePassIE): 'tveToken': tve_token, }) + if content_type == 'playlist': + return self._extract_playlist(json_data, headers) + return self._parse_brightcove_metadata( json_data, video_id, headers=headers) From cb434d5f3889549aee02ad935136f5c867e92db0 Mon Sep 17 00:00:00 2001 From: Kyle Date: Sun, 23 Jun 2019 09:17:27 +0900 Subject: [PATCH 3/4] Add requested changes. --- youtube_dl/extractor/brightcove.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index a0a11628f..ca31501ee 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -517,18 +517,20 @@ class BrightcoveNewIE(AdobePassIE): 'skip_download': True, } }, { - # playlist streams - 'url': 'http://players.brightcove.net/5690807595001/HyZNerRl7_default/index.html?playlistId=5743160747001', + # playlist stream + 'url': 'https://players.brightcove.net/1752604059001/S13cJdUBz_default/index.html?playlistId=5718313430001', 'info_dict': { - 'id': '5743160747001', - 'ext': 'mp4', - 'title': 'prod ntv', - 'uploader_id': '5690807595001', + 'id': '5718313430001', + 'title': 'No Audio Playlist', }, + 'playlist_count': 7, 'params': { # m3u8 download 'skip_download': True, } + }, { + 'url': 'http://players.brightcove.net/5690807595001/HyZNerRl7_default/index.html?playlistId=5743160747001', + 'only_matching': True, }, { # ref: prefixed video id 'url': 'http://players.brightcove.net/3910869709001/21519b5c-4b3b-4363-accb-bdc8f358f823_default/index.html?videoId=ref:7069442', @@ -721,17 +723,6 @@ class BrightcoveNewIE(AdobePassIE): 'is_live': is_live, } - def _extract_playlist(self, json_data, headers): - playlist_id = json_data.get('id') - playlist_title = json_data.get('name') - playlist_description = json_data.get('description') - playlist_vids = json_data.get('videos', []) - - return self.playlist_result( - [self._parse_brightcove_metadata(vid, vid.get('id'), headers) for vid in playlist_vids], - playlist_id, playlist_title, playlist_description, - ) - def _real_extract(self, url): url, smuggled_data = unsmuggle_url(url, {}) self._initialize_geo_bypass({ @@ -796,7 +787,10 @@ class BrightcoveNewIE(AdobePassIE): }) if content_type == 'playlist': - return self._extract_playlist(json_data, headers) + return self.playlist_result( + [self._parse_brightcove_metadata(vid, vid.get('id'), headers) for vid in json_data.get('videos', []) if vid.get('id')], + json_data.get('id'), json_data.get('name'), json_data.get('description'), + ) return self._parse_brightcove_metadata( json_data, video_id, headers=headers) From 9985e268a9d70357add23b6bfcce2789bb9516e8 Mon Sep 17 00:00:00 2001 From: Sergey M Date: Sun, 23 Jun 2019 17:03:00 +0700 Subject: [PATCH 4/4] Update brightcove.py --- youtube_dl/extractor/brightcove.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index ca31501ee..58ec5c979 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -788,9 +788,10 @@ class BrightcoveNewIE(AdobePassIE): if content_type == 'playlist': return self.playlist_result( - [self._parse_brightcove_metadata(vid, vid.get('id'), headers) for vid in json_data.get('videos', []) if vid.get('id')], - json_data.get('id'), json_data.get('name'), json_data.get('description'), - ) + [self._parse_brightcove_metadata(vid, vid.get('id'), headers) + for vid in json_data.get('videos', []) if vid.get('id')], + json_data.get('id'), json_data.get('name'), + json_data.get('description')) return self._parse_brightcove_metadata( json_data, video_id, headers=headers)