From b2247d51255135bc1a0551613a01406b65b7aadb Mon Sep 17 00:00:00 2001 From: Thomas van der Berg Date: Wed, 31 Jan 2018 23:00:30 +0100 Subject: [PATCH 1/2] [vtv] Add new extractor --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/vtv.py | 86 ++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 youtube_dl/extractor/vtv.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index b442256fe..38f7184bc 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1275,6 +1275,7 @@ from .vrv import ( VRVSeriesIE, ) from .vshare import VShareIE +from .vtv import VTVIE from .medialaan import MedialaanIE from .vube import VubeIE from .vuclip import VuClipIE diff --git a/youtube_dl/extractor/vtv.py b/youtube_dl/extractor/vtv.py new file mode 100644 index 000000000..2ebfbf8a7 --- /dev/null +++ b/youtube_dl/extractor/vtv.py @@ -0,0 +1,86 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + +import re + +from ..utils import extract_attributes + +class VTVIE(InfoExtractor): + _VALID_URL = r'https?://..\.tvnet\.gov\.vn/[^/]*/(?P[0-9]+)/?.*' + _TESTS = [{ + 'url': 'http://us.tvnet.gov.vn/kenh-truyen-hinh/1011/vtv1', + 'info_dict': { + 'id': '1011', + 'ext': 'mp4', + 'title': r're:VTV1 | LiveTV - TV Net .*', + 'thumbnail': r're:https?://.*\.png$', + } + }, { + 'url': 'http://us.tvnet.gov.vn/video/109788/vtv1---bac-tuyet-tai-lao-cai-va-ha-giang/tin-nong-24h', + 'md5': '5263c63d738569ed507980f1e49ebc03', + 'info_dict': { + 'id': '109788', + 'ext': 'mp4', + 'title': 'VTV1 - Bắc tuyết tại Lào Cai và Hà Giang - TV Net', + 'thumbnail': r're:https?://.*\.JPG$', + } + }, { + 'url': 'http://us.tvnet.gov.vn/kenh-truyen-hinh/1014', + 'info_dict': { + 'id': '1014', + 'ext': 'm4a', + 'title': r're:VOV1 | LiveTV - TV Net .*', + 'thumbnail': r're:https?://.*\.png$', + } + + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + title = self._html_search_regex(r'(.+?)', webpage, 'title', fatal=False) + if title is None: + title = self._og_search_title(webpage) + title.strip() + + mediaplayer_div = self._search_regex(r'(]*id="mediaplayer"[^>]*>)', webpage, 'mediaplayer element') + mediaplayer_div_attributes = extract_attributes(mediaplayer_div) + + thumbnail = mediaplayer_div_attributes.get("data-image") + + json_url = mediaplayer_div_attributes.get("data-file") + video_streams = self._download_json(json_url, video_id) + + # little hack to better support radio streams + if title.startswith("VOV"): + ext = "m4a" + else: + ext = "mp4" + + # get any working playlist from streams. Currently there's 2 and the first always works, + # but you never know in the future + for stream in video_streams: + formats = self._extract_m3u8_formats(stream.get("url"), video_id, ext=ext, fatal=False) + if len(formats) != 0: + break + + if re.match(r'https?://[^/]*/video/.*', url) is not None: + is_live = False + elif re.match(r'https?://[^/]*/kenh-truyen-hinh/.*', url) is not None: + is_live = True + else: + is_live = None + + if is_live: + title = self._live_title(title) + + return { + 'id': video_id, + 'title': title, + 'thumbnail': thumbnail, + 'formats': formats, + 'is_live': is_live, + } From c045997cbb5d890d0c769a2681236567ae9774c7 Mon Sep 17 00:00:00 2001 From: Thomas van der Berg Date: Thu, 8 Feb 2018 23:19:28 +0100 Subject: [PATCH 2/2] [VTV] Improve based on code review --- youtube_dl/extractor/vtv.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/youtube_dl/extractor/vtv.py b/youtube_dl/extractor/vtv.py index 2ebfbf8a7..a9683dd85 100644 --- a/youtube_dl/extractor/vtv.py +++ b/youtube_dl/extractor/vtv.py @@ -8,17 +8,19 @@ import re from ..utils import extract_attributes class VTVIE(InfoExtractor): - _VALID_URL = r'https?://..\.tvnet\.gov\.vn/[^/]*/(?P[0-9]+)/?.*' + _VALID_URL = r'https?://(au|ca|cz|de|jp|kr|tw|us|vn)\.tvnet\.gov\.vn/[^/]*/(?P[0-9]+)/?' _TESTS = [{ + # Livestream. Channel: VTV 1 'url': 'http://us.tvnet.gov.vn/kenh-truyen-hinh/1011/vtv1', 'info_dict': { 'id': '1011', 'ext': 'mp4', - 'title': r're:VTV1 | LiveTV - TV Net .*', + 'title': r're:^VTV1 | LiveTV - TV Net [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', 'thumbnail': r're:https?://.*\.png$', } }, { - 'url': 'http://us.tvnet.gov.vn/video/109788/vtv1---bac-tuyet-tai-lao-cai-va-ha-giang/tin-nong-24h', + # Downloading a video. + 'url': 'http://de.tvnet.gov.vn/video/109788/vtv1---bac-tuyet-tai-lao-cai-va-ha-giang/tin-nong-24h', 'md5': '5263c63d738569ed507980f1e49ebc03', 'info_dict': { 'id': '109788', @@ -27,11 +29,13 @@ class VTVIE(InfoExtractor): 'thumbnail': r're:https?://.*\.JPG$', } }, { - 'url': 'http://us.tvnet.gov.vn/kenh-truyen-hinh/1014', + # Radio live stream. Channel: VOV 1 + 'url': 'http://vn.tvnet.gov.vn/kenh-truyen-hinh/1014', 'info_dict': { 'id': '1014', 'ext': 'm4a', - 'title': r're:VOV1 | LiveTV - TV Net .*', + 'vcodec': 'none', + 'title': r're:VOV1 | LiveTV - TV Net [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', 'thumbnail': r're:https?://.*\.png$', } @@ -41,7 +45,7 @@ class VTVIE(InfoExtractor): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - title = self._html_search_regex(r'(.+?)', webpage, 'title', fatal=False) + title = self._html_search_regex(r'(.+?)', webpage, 'title', default=None, fatal=False) if title is None: title = self._og_search_title(webpage) title.strip() @@ -51,25 +55,26 @@ class VTVIE(InfoExtractor): thumbnail = mediaplayer_div_attributes.get("data-image") - json_url = mediaplayer_div_attributes.get("data-file") + json_url = mediaplayer_div_attributes["data-file"] video_streams = self._download_json(json_url, video_id) - # little hack to better support radio streams - if title.startswith("VOV"): - ext = "m4a" - else: - ext = "mp4" # get any working playlist from streams. Currently there's 2 and the first always works, # but you never know in the future for stream in video_streams: - formats = self._extract_m3u8_formats(stream.get("url"), video_id, ext=ext, fatal=False) - if len(formats) != 0: + formats = self._extract_m3u8_formats(stream.get("url"), video_id, ext="mp4", fatal=False) + if formats: break - if re.match(r'https?://[^/]*/video/.*', url) is not None: + # better support radio streams + if title.startswith("VOV"): + for f in formats: + f["ext"] = "m4a" + f["vcodec"] = "none" + + if "/video/" in url or "/radio/" in url: is_live = False - elif re.match(r'https?://[^/]*/kenh-truyen-hinh/.*', url) is not None: + elif "/kenh-truyen-hinh/" in url: is_live = True else: is_live = None