diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md index 3a94bd621..e6b82fda3 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE/1_broken_site.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dl version **2019.11.28** +- [ ] I've verified that I'm running youtube-dl version **2019.12.25** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2019.11.28 + [debug] youtube-dl version 2019.12.25 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md index 72bee12aa..9096af717 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md @@ -19,7 +19,7 @@ labels: 'site-support-request' - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dl version **2019.11.28** +- [ ] I've verified that I'm running youtube-dl version **2019.12.25** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md index ddf67e951..5c235df0a 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -18,13 +18,13 @@ title: '' - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dl version **2019.11.28** +- [ ] I've verified that I'm running youtube-dl version **2019.12.25** - [ ] I've searched the bugtracker for similar site feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md index 7122e2714..fe6ab9aa0 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE/4_bug_report.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dl version **2019.11.28** +- [ ] I've verified that I'm running youtube-dl version **2019.12.25** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2019.11.28 + [debug] youtube-dl version 2019.12.25 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md index a93882b39..76b028de4 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE/5_feature_request.md @@ -19,13 +19,13 @@ labels: 'request' - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dl version **2019.11.28** +- [ ] I've verified that I'm running youtube-dl version **2019.12.25** - [ ] I've searched the bugtracker for similar feature requests including closed ones diff --git a/ChangeLog b/ChangeLog index d2f17ee06..18080575b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,10 +1,11 @@ -version +version 2019.12.25 Core * [utils] Improve str_to_int + [downloader/hls] Add ability to override AES decryption key URL (#17521) Extractors +* [mediaset] Fix parse formats (#23508) + [tv2dk:bornholm:play] Add support for play.tv2bornholm.dk (#23291) + [slideslive] Add support for url and vimeo service names (#23414) * [slideslive] Fix extraction (#23413) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 2744dfca8..04956c546 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -28,10 +28,11 @@ - **acast:channel** - **ADN**: Anime Digital Network - **AdobeConnect** - - **AdobeTV** - - **AdobeTVChannel** - - **AdobeTVShow** - - **AdobeTVVideo** + - **adobetv** + - **adobetv:channel** + - **adobetv:embed** + - **adobetv:show** + - **adobetv:video** - **AdultSwim** - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault - **afreecatv**: afreecatv.com @@ -396,6 +397,7 @@ - **Kankan** - **Karaoketv** - **KarriereVideos** + - **Katsomo** - **KeezMovies** - **Ketnet** - **KhanAcademy** @@ -403,7 +405,6 @@ - **KinjaEmbed** - **KinoPoisk** - **KonserthusetPlay** - - **kontrtube**: KontrTube.ru - Труба зовёт - **KrasView**: Красвью - **Ku6** - **KUSI** @@ -513,7 +514,6 @@ - **mtvjapan** - **mtvservices:embedded** - **MuenchenTV**: münchen.tv - - **MusicPlayOn** - **mva**: Microsoft Virtual Academy videos - **mva:course**: Microsoft Virtual Academy courses - **Mwave** @@ -913,6 +913,7 @@ - **tv2.hu** - **TV2Article** - **TV2DK** + - **TV2DKBornholmPlay** - **TV4**: tv4.se and tv4play.se - **TV5MondePlus**: TV5MONDE+ - **TVA** @@ -954,6 +955,7 @@ - **udemy** - **udemy:course** - **UDNEmbed**: 聯合影音 + - **UFCArabia** - **UFCTV** - **UKTVPlay** - **umg:de**: Universal Music Deutschland @@ -993,7 +995,6 @@ - **videomore** - **videomore:season** - **videomore:video** - - **VideoPremium** - **VideoPress** - **Vidio** - **VidLii** diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py index 6b0e64b7f..50234798b 100644 --- a/youtube_dl/extractor/mailru.py +++ b/youtube_dl/extractor/mailru.py @@ -20,10 +20,10 @@ class MailRuIE(InfoExtractor): IE_DESC = 'Видео@Mail.Ru' _VALID_URL = r'''(?x) https?:// - (?:(?:www|m)\.)?my\.mail\.ru/ + (?:(?:www|m)\.)?my\.mail\.ru/+ (?: video/.*\#video=/?(?P(?:[^/]+/){3}\d+)| - (?:(?P(?:[^/]+/){2})video/(?P[^/]+/\d+))\.html| + (?:(?P(?:[^/]+/+){2})video/(?P[^/]+/\d+))\.html| (?:video/embed|\+/video/meta)/(?P\d+) ) ''' @@ -85,6 +85,14 @@ class MailRuIE(InfoExtractor): { 'url': 'http://my.mail.ru/+/video/meta/7949340477499637815', 'only_matching': True, + }, + { + 'url': 'https://my.mail.ru//list/sinyutin10/video/_myvideo/4.html', + 'only_matching': True, + }, + { + 'url': 'https://my.mail.ru//list//sinyutin10/video/_myvideo/4.html', + 'only_matching': True, } ] @@ -237,7 +245,7 @@ class MailRuMusicSearchBaseIE(InfoExtractor): class MailRuMusicIE(MailRuMusicSearchBaseIE): IE_NAME = 'mailru:music' IE_DESC = 'Музыка@Mail.Ru' - _VALID_URL = r'https?://my\.mail\.ru/music/songs/[^/?#&]+-(?P[\da-f]+)' + _VALID_URL = r'https?://my\.mail\.ru/+music/+songs/+[^/?#&]+-(?P[\da-f]+)' _TESTS = [{ 'url': 'https://my.mail.ru/music/songs/%D0%BC8%D0%BB8%D1%82%D1%85-l-a-h-luciferian-aesthetics-of-herrschaft-single-2017-4e31f7125d0dfaef505d947642366893', 'md5': '0f8c22ef8c5d665b13ac709e63025610', @@ -273,7 +281,7 @@ class MailRuMusicIE(MailRuMusicSearchBaseIE): class MailRuMusicSearchIE(MailRuMusicSearchBaseIE): IE_NAME = 'mailru:music:search' IE_DESC = 'Музыка@Mail.Ru' - _VALID_URL = r'https?://my\.mail\.ru/music/search/(?P[^/?#&]+)' + _VALID_URL = r'https?://my\.mail\.ru/+music/+search/+(?P[^/?#&]+)' _TESTS = [{ 'url': 'https://my.mail.ru/music/search/black%20shadow', 'info_dict': { diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py index f976506f4..027a790b8 100644 --- a/youtube_dl/extractor/mediaset.py +++ b/youtube_dl/extractor/mediaset.py @@ -123,7 +123,7 @@ class MediasetIE(ThePlatformBaseIE): def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None): for video in smil.findall(self._xpath_ns('.//video', namespace)): video.attrib['src'] = re.sub(r'(https?://vod05)t(-mediaset-it\.akamaized\.net/.+?.mpd)\?.+', r'\1\2', video.attrib['src']) - return super()._parse_smil_formats(smil, smil_url, video_id, namespace, f4m_params, transform_rtmp_url) + return super(MediasetIE, self)._parse_smil_formats(smil, smil_url, video_id, namespace, f4m_params, transform_rtmp_url) def _real_extract(self, url): guid = self._match_id(url) diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py index 40f214a87..ad9da9612 100644 --- a/youtube_dl/extractor/mitele.py +++ b/youtube_dl/extractor/mitele.py @@ -4,8 +4,8 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..utils import ( int_or_none, + parse_iso8601, smuggle_url, - parse_duration, ) @@ -18,16 +18,18 @@ class MiTeleIE(InfoExtractor): 'info_dict': { 'id': 'FhYW1iNTE6J6H7NkQRIEzfne6t2quqPg', 'ext': 'mp4', - 'title': 'Tor, la web invisible', - 'description': 'md5:3b6fce7eaa41b2d97358726378d9369f', + 'title': 'Diario de La redacción Programa 144', + 'description': 'md5:07c35a7b11abb05876a6a79185b58d27', 'series': 'Diario de', - 'season': 'La redacción', + 'season': 'Season 14', 'season_number': 14, - 'season_id': 'diario_de_t14_11981', - 'episode': 'Programa 144', + 'episode': 'Tor, la web invisible', 'episode_number': 3, 'thumbnail': r're:(?i)^https?://.*\.jpg$', 'duration': 2913, + 'age_limit': 16, + 'timestamp': 1471209401, + 'upload_date': '20160814', }, 'add_ie': ['Ooyala'], }, { @@ -39,13 +41,15 @@ class MiTeleIE(InfoExtractor): 'title': 'Cuarto Milenio Temporada 6 Programa 226', 'description': 'md5:5ff132013f0cd968ffbf1f5f3538a65f', 'series': 'Cuarto Milenio', - 'season': 'Temporada 6', + 'season': 'Season 6', 'season_number': 6, - 'season_id': 'cuarto_milenio_t06_12715', - 'episode': 'Programa 226', + 'episode': 'Episode 24', 'episode_number': 24, 'thumbnail': r're:(?i)^https?://.*\.jpg$', 'duration': 7313, + 'age_limit': 12, + 'timestamp': 1471209021, + 'upload_date': '20160814', }, 'params': { 'skip_download': True, @@ -54,67 +58,36 @@ class MiTeleIE(InfoExtractor): }, { 'url': 'http://www.mitele.es/series-online/la-que-se-avecina/57aac5c1c915da951a8b45ed/player', 'only_matching': True, + }, { + 'url': 'https://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144-40_1006364575251/player/', + 'only_matching': True, }] def _real_extract(self, url): - video_id = self._match_id(url) - - paths = self._download_json( - 'https://www.mitele.es/amd/agp/web/metadata/general_configuration', - video_id, 'Downloading paths JSON') - - ooyala_s = paths['general_configuration']['api_configuration']['ooyala_search'] - base_url = ooyala_s.get('base_url', 'cdn-search-mediaset.carbyne.ps.ooyala.com') - full_path = ooyala_s.get('full_path', '/search/v1/full/providers/') - source = self._download_json( - '%s://%s%s%s/docs/%s' % ( - ooyala_s.get('protocol', 'https'), base_url, full_path, - ooyala_s.get('provider_id', '104951'), video_id), - video_id, 'Downloading data JSON', query={ - 'include_titles': 'Series,Season', - 'product_name': ooyala_s.get('product_name', 'test'), - 'format': 'full', - })['hits']['hits'][0]['_source'] - - embedCode = source['offers'][0]['embed_codes'][0] - titles = source['localizable_titles'][0] - - title = titles.get('title_medium') or titles['title_long'] - - description = titles.get('summary_long') or titles.get('summary_medium') - - def get(key1, key2): - value1 = source.get(key1) - if not value1 or not isinstance(value1, list): - return - if not isinstance(value1[0], dict): - return - return value1[0].get(key2) - - series = get('localizable_titles_series', 'title_medium') - - season = get('localizable_titles_season', 'title_medium') - season_number = int_or_none(source.get('season_number')) - season_id = source.get('season_id') - - episode = titles.get('title_sort_name') - episode_number = int_or_none(source.get('episode_number')) - - duration = parse_duration(get('videos', 'duration')) + display_id = self._match_id(url) + webpage = self._download_webpage(url, display_id) + pre_player = self._parse_json(self._search_regex( + r'window\.\$REACTBASE_STATE\.prePlayer_mtweb\s*=\s*({.+})', + webpage, 'Pre Player'), display_id)['prePlayer'] + title = pre_player['title'] + video = pre_player['video'] + video_id = video['dataMediaId'] + content = pre_player.get('content') or {} + info = content.get('info') or {} return { '_type': 'url_transparent', # for some reason only HLS is supported - 'url': smuggle_url('ooyala:' + embedCode, {'supportedformats': 'm3u8,dash'}), + 'url': smuggle_url('ooyala:' + video_id, {'supportedformats': 'm3u8,dash'}), 'id': video_id, 'title': title, - 'description': description, - 'series': series, - 'season': season, - 'season_number': season_number, - 'season_id': season_id, - 'episode': episode, - 'episode_number': episode_number, - 'duration': duration, - 'thumbnail': get('images', 'url'), + 'description': info.get('synopsis'), + 'series': content.get('title'), + 'season_number': int_or_none(info.get('season_number')), + 'episode': content.get('subtitle'), + 'episode_number': int_or_none(info.get('episode_number')), + 'duration': int_or_none(info.get('duration')), + 'thumbnail': video.get('dataPoster'), + 'age_limit': int_or_none(info.get('rating')), + 'timestamp': parse_iso8601(pre_player.get('publishedTime')), } diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dl/extractor/scrippsnetworks.py index 8b3275735..afab9591d 100644 --- a/youtube_dl/extractor/scrippsnetworks.py +++ b/youtube_dl/extractor/scrippsnetworks.py @@ -7,6 +7,7 @@ import re from .aws import AWSIE from .anvato import AnvatoIE +from .common import InfoExtractor from ..utils import ( smuggle_url, urlencode_postdata, @@ -102,3 +103,46 @@ class ScrippsNetworksWatchIE(AWSIE): 'anvato:anvato_scripps_app_web_prod_0837996dbe373629133857ae9eb72e740424d80a:%s' % mcp_id, {'geo_countries': ['US']}), AnvatoIE.ie_key(), video_id=mcp_id) + + +class ScrippsNetworksIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?(?Pcookingchanneltv|(?:diy|food)network|hgtv|travelchannel)\.com/videos/[0-9a-z-]+-(?P\d+)' + _TESTS = [{ + 'url': 'https://www.cookingchanneltv.com/videos/the-best-of-the-best-0260338', + 'info_dict': { + 'id': '0260338', + 'ext': 'mp4', + 'title': 'The Best of the Best', + 'description': 'Catch a new episode of MasterChef Canada Tuedsay at 9/8c.', + 'timestamp': 1475678834, + 'upload_date': '20161005', + 'uploader': 'SCNI-SCND', + }, + 'add_ie': ['ThePlatform'], + }, { + 'url': 'https://www.diynetwork.com/videos/diy-barnwood-tablet-stand-0265790', + 'only_matching': True, + }, { + 'url': 'https://www.foodnetwork.com/videos/chocolate-strawberry-cake-roll-7524591', + 'only_matching': True, + }, { + 'url': 'https://www.hgtv.com/videos/cookie-decorating-101-0301929', + 'only_matching': True, + }, { + 'url': 'https://www.travelchannel.com/videos/two-climates-one-bag-5302184', + 'only_matching': True, + }] + _ACCOUNT_MAP = { + 'cookingchanneltv': 2433005105, + 'diynetwork': 2433004575, + 'foodnetwork': 2433005105, + 'hgtv': 2433004575, + 'travelchannel': 2433005739, + } + _TP_TEMPL = 'https://link.theplatform.com/s/ip77QC/media/guid/%d/%s?mbr=true' + + def _real_extract(self, url): + site, guid = re.match(self._VALID_URL, url).groups() + return self.url_result(smuggle_url( + self._TP_TEMPL % (self._ACCOUNT_MAP[site], guid), + {'force_smil_url': True}), 'ThePlatform', guid) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index c2ee54457..2128e5957 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -255,7 +255,7 @@ class SoundcloudIE(InfoExtractor): _API_BASE = 'https://api.soundcloud.com/' _API_V2_BASE = 'https://api-v2.soundcloud.com/' _BASE_URL = 'https://soundcloud.com/' - _CLIENT_ID = 'UW9ajvMgVdMMW3cdeBi8lPfN6dvOVGji' + _CLIENT_ID = 'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk' _IMAGE_REPL_RE = r'-([0-9a-z]+)\.jpg' _ARTWORK_MAP = { diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 1227abc0a..606dbe1fb 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2019.11.28' +__version__ = '2019.12.25'