From 9d81c7d78cc3ac477a19cda84e7649d025399e23 Mon Sep 17 00:00:00 2001 From: remitamine Date: Sun, 13 Mar 2016 11:05:38 +0100 Subject: [PATCH] return mpd_url for dash segmented formats(#8704) mpd_url construction rules can be found in Annex C section 4 in ISO/IEC 23009-1 - separate downloader specific params from other info_dict keys - the mpd_url can be used with external programs like MP4Box and VLC 3 - it's needed to be passed to dash downloader to support live stream download --- youtube_dl/downloader/dash.py | 7 ++++--- youtube_dl/extractor/common.py | 33 +++++++++++++++++++++++++-------- youtube_dl/extractor/generic.py | 2 +- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py index 8b1b17c6e..4fb2219e8 100644 --- a/youtube_dl/downloader/dash.py +++ b/youtube_dl/downloader/dash.py @@ -18,9 +18,10 @@ class DashSegmentsFD(FragmentFD): FD_NAME = 'dashsegments' def real_download(self, filename, info_dict): - base_url = info_dict['url'] - segment_urls = [info_dict['segment_urls'][0]] if self.params.get('test', False) else info_dict['segment_urls'] - initialization_url = info_dict.get('initialization_url') + params = info_dict['_downloader_params'] + base_url = params['base_url'] + segment_urls = [params['segment_urls'][0]] if self.params.get('test', False) else params['segment_urls'] + initialization_url = params.get('initialization_url') ctx = { 'filename': filename, diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index ecd7da767..c7bc6a51c 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -23,6 +23,7 @@ from ..compat import ( compat_urllib_error, compat_urllib_parse, compat_urlparse, + compat_urllib_parse_urlparse, ) from ..utils import ( NO_DEFAULT, @@ -1386,12 +1387,11 @@ class InfoExtractor(object): if res is False: return [] mpd, urlh = res - mpd_base_url = re.match(r'https?://.+/', urlh.geturl()).group() return self._parse_mpd_formats( - compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url, formats_dict=formats_dict) + compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, urlh.geturl(), formats_dict=formats_dict) - def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}): + def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_url='', formats_dict={}): if mpd_doc.get('type') == 'dynamic': return [] @@ -1445,6 +1445,10 @@ class InfoExtractor(object): ms_info['initialization_url'] = initialization.attrib['sourceURL'] return ms_info + mpd_base_url = mpd_url.rpartition('/')[0] + parsed_mpd_url = None + if mpd_base_url: + parsed_mpd_url = compat_urllib_parse_urlparse(mpd_url) mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration')) formats = [] for period in mpd_doc.findall(_add_ns('Period')): @@ -1509,18 +1513,31 @@ class InfoExtractor(object): media_template = re.sub(r'\$(Number|Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template) media_template.replace('$$', '$') representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number, 'Bandwidth': representation_attrib.get('bandwidth')} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])] - if 'segment_urls' in representation_ms_info: + segment_urls = representation_ms_info.get('segment_urls') + if segment_urls: + fragment_mpd_url = segment_urls[0] + if parsed_mpd_url: + fragment_parts = ((period, 'id', 'period'), (adaptation_set, 'id', 'as'), (adaptation_set, 'group', 'track')) + params = {} + for ele, attr, frag_attr in fragment_parts: + frag_val = ele.attrib.get(attr) + if frag_val: + params[frag_attr] = frag_val + fragment = compat_urllib_parse.urlencode(params) + fragment_mpd_url = parsed_mpd_url._replace(fragment=fragment).geturl() f.update({ - 'segment_urls': representation_ms_info['segment_urls'], + 'url': fragment_mpd_url, + '_downloader_params': { + 'segment_urls': segment_urls, + 'base_url': base_url, + }, 'protocol': 'http_dash_segments', }) if 'initialization_url' in representation_ms_info: initialization_url = representation_ms_info['initialization_url'].replace('$RepresentationID$', representation_id) - f.update({ + f['_downloader_params'].update({ 'initialization_url': initialization_url, }) - if not f.get('url'): - f['url'] = initialization_url try: existing_format = next( fo for fo in formats diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 8121f04a5..0d14a04ad 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -1320,7 +1320,7 @@ class GenericIE(InfoExtractor): return self.playlist_result(self._parse_xspf(doc, video_id), video_id) elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag): info_dict['formats'] = self._parse_mpd_formats( - doc, video_id, mpd_base_url=url.rpartition('/')[0]) + doc, video_id, mpd_url=url) return info_dict elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag): info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)