From 3d10b1b8a6fd3342f6b389affdbe984c87ba2f9e Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Sun, 21 Jul 2019 00:24:01 +0200 Subject: [PATCH 01/10] Add support for raiplayradio.it --- docs/supportedsites.md | 2 + youtube_dl/extractor/extractors.py | 2 + youtube_dl/extractor/rai.py | 99 ++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 9ae6e5c96..afd34f785 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -718,6 +718,8 @@ - **RaiPlay** - **RaiPlayLive** - **RaiPlayPlaylist** + - **RaiPlayRadio** + - **RaiPlayRadioPlaylist** - **RayWenderlich** - **RayWenderlichCourse** - **RBMARadio** diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 15f54a214..d313bb89e 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -919,6 +919,8 @@ from .rai import ( RaiPlayLiveIE, RaiPlayPlaylistIE, RaiIE, + RaiPlayRadioIE, + RaiPlayRadioPlaylistIE, ) from .raywenderlich import ( RayWenderlichIE, diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py index 207a6c247..7cf97bb6c 100644 --- a/youtube_dl/extractor/rai.py +++ b/youtube_dl/extractor/rai.py @@ -4,6 +4,7 @@ import re from .common import InfoExtractor from ..compat import ( + compat_HTMLParser, compat_urlparse, compat_str, ) @@ -13,6 +14,7 @@ from ..utils import ( find_xpath_attr, fix_xml_ampersands, GeoRestrictedError, + get_element_by_class, int_or_none, parse_duration, strip_or_none, @@ -500,3 +502,100 @@ class RaiIE(RaiBaseIE): info.update(relinker_info) return info + + +class HTMLListAttrsParser(compat_HTMLParser): + def __init__(self): + compat_HTMLParser.__init__(self) + self.items = [] + self._level = 0 + + def handle_starttag(self, tag, attrs): + if tag == 'li' and self._level == 0: + self.items.append(dict(attrs)) + self._level += 1 + + def handle_endtag(self, tag): + self._level -= 1 + + +class RaiPlayRadioBaseIE(InfoExtractor): + _BASE = 'https://www.raiplayradio.it' + + def parse_list(self, webpage): + parser = HTMLListAttrsParser() + parser.feed(webpage) + parser.close() + return parser.items + + def get_playlist_iter(self, url): + webpage = self._download_webpage(url, url) + for attrs in self.parse_list(webpage): + title = attrs['data-title'].strip() + webpage = urljoin(url, attrs['data-mediapolis']) + audio_url = compat_str(self._request_webpage(webpage, title).geturl()) + yield { + 'url': audio_url, + 'id': attrs['data-uniquename'].lstrip('ContentItem-'), + 'title': title, + 'ext': determine_ext(audio_url), + 'thumbnail': urljoin(url, attrs['data-image']), + 'language': 'it', + } + + def get_playlist(self, *args, **kwargs): + return list(self.get_playlist_iter(*args, **kwargs)) + + +class RaiPlayRadioIE(RaiPlayRadioBaseIE): + _VALID_URL = r'%s/audio/.+?-(?P%s)\.html' % (RaiPlayRadioBaseIE._BASE, RaiBaseIE._UUID_RE) + _TEST = { + 'url': 'https://www.raiplayradio.it/audio/2019/07/RADIO3---LEZIONI-DI-MUSICA-36b099ff-4123-4443-9bf9-38e43ef5e025.html', + 'info_dict': { + 'id': '36b099ff-4123-4443-9bf9-38e43ef5e025', + 'ext': 'mp3', + 'title': 'Dal "Chiaro di luna" al "Clair de lune", prima parte con Giovanni Bietti', + 'thumbnail': 'https://www.raiplayradio.it/cropgd/400x400/dl/img/2019/07/08/1562590329054_luna.jpg', + 'language': 'it', + } + } + + def _real_extract(self, url): + list_url = url.replace('.html', '-list.html') + for entry in self.get_playlist_iter(list_url): + if entry['id'] == self._match_id(url): + return entry + + +class RaiPlayRadioPlaylistIE(RaiPlayRadioBaseIE): + _VALID_URL = r'%s/playlist/.+?-(?P%s)\.html' % (RaiPlayRadioBaseIE._BASE, RaiBaseIE._UUID_RE) + _TEST = { + 'url': 'https://www.raiplayradio.it/playlist/2017/12/Alice-nel-paese-delle-meraviglie-72371d3c-d998-49f3-8860-d168cfdf4966.html', + 'info_dict': { + 'id': '72371d3c-d998-49f3-8860-d168cfdf4966', + 'title': "Alice nel paese delle meraviglie", + 'description': "di Lewis Carrol letto da Aldo Busi", + }, + 'playlist_count': 11, + } + + def _real_extract(self, url): + playlist_id = self._match_id(url) + playlist_webpage = self._download_webpage(url, playlist_id) + playlist_title = unescapeHTML(self._html_search_regex(r'data-playlist-title="(.+?)"', playlist_webpage, 'title')) + playlist_creator = self._html_search_meta('nomeProgramma', playlist_webpage) + playlist_description = get_element_by_class('textDescriptionProgramma', playlist_webpage) + + player_href = self._html_search_regex(r'data-player-href="(.+?)"', playlist_webpage, 'href') + list_url = urljoin(url, player_href) + + entries = self.get_playlist(list_url) + for index, entry in enumerate(entries, start=1): + entry.update({ + 'track': entry['title'], + 'track_number': index, + 'album': playlist_title, + 'artist': playlist_creator, + }) + + return self.playlist_result(entries, playlist_id, playlist_title, playlist_description) From 5a3905826ac9bb71e37d9798c27853f6eed081a5 Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Sat, 21 Sep 2019 15:06:29 +0200 Subject: [PATCH 02/10] Fix raiplayradio.it code based on coding conventions --- youtube_dl/extractor/rai.py | 54 +++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py index 7cf97bb6c..a758d5653 100644 --- a/youtube_dl/extractor/rai.py +++ b/youtube_dl/extractor/rai.py @@ -533,32 +533,35 @@ class RaiPlayRadioBaseIE(InfoExtractor): for attrs in self.parse_list(webpage): title = attrs['data-title'].strip() webpage = urljoin(url, attrs['data-mediapolis']) - audio_url = compat_str(self._request_webpage(webpage, title).geturl()) + audio_url = compat_str(self._request_webpage( + webpage, title).geturl()) yield { 'url': audio_url, 'id': attrs['data-uniquename'].lstrip('ContentItem-'), 'title': title, 'ext': determine_ext(audio_url), 'thumbnail': urljoin(url, attrs['data-image']), - 'language': 'it', - } + 'language': 'it'} def get_playlist(self, *args, **kwargs): return list(self.get_playlist_iter(*args, **kwargs)) class RaiPlayRadioIE(RaiPlayRadioBaseIE): - _VALID_URL = r'%s/audio/.+?-(?P%s)\.html' % (RaiPlayRadioBaseIE._BASE, RaiBaseIE._UUID_RE) + _VALID_URL = r'%s/audio/.+?-(?P%s)\.html' % ( + RaiPlayRadioBaseIE._BASE, RaiBaseIE._UUID_RE) _TEST = { - 'url': 'https://www.raiplayradio.it/audio/2019/07/RADIO3---LEZIONI-DI-MUSICA-36b099ff-4123-4443-9bf9-38e43ef5e025.html', + 'url': 'https://www.raiplayradio.it/audio/2019/07/' + 'RADIO3---LEZIONI-DI-MUSICA-' + '36b099ff-4123-4443-9bf9-38e43ef5e025.html', 'info_dict': { 'id': '36b099ff-4123-4443-9bf9-38e43ef5e025', 'ext': 'mp3', - 'title': 'Dal "Chiaro di luna" al "Clair de lune", prima parte con Giovanni Bietti', - 'thumbnail': 'https://www.raiplayradio.it/cropgd/400x400/dl/img/2019/07/08/1562590329054_luna.jpg', - 'language': 'it', - } - } + 'title': 'Dal "Chiaro di luna" al "Clair de lune", ' + 'prima parte con Giovanni Bietti', + 'thumbnail': 'https://www.raiplayradio.it/cropgd/400x400/dl/' + 'img/2019/07/08/1562590329054_luna.jpg', + 'language': 'it'}} def _real_extract(self, url): list_url = url.replace('.html', '-list.html') @@ -568,25 +571,30 @@ class RaiPlayRadioIE(RaiPlayRadioBaseIE): class RaiPlayRadioPlaylistIE(RaiPlayRadioBaseIE): - _VALID_URL = r'%s/playlist/.+?-(?P%s)\.html' % (RaiPlayRadioBaseIE._BASE, RaiBaseIE._UUID_RE) + _VALID_URL = r'%s/playlist/.+?-(?P%s)\.html' % ( + RaiPlayRadioBaseIE._BASE, RaiBaseIE._UUID_RE) _TEST = { - 'url': 'https://www.raiplayradio.it/playlist/2017/12/Alice-nel-paese-delle-meraviglie-72371d3c-d998-49f3-8860-d168cfdf4966.html', + 'url': 'https://www.raiplayradio.it/playlist/2017/12/' + 'Alice-nel-paese-delle-meraviglie-' + '72371d3c-d998-49f3-8860-d168cfdf4966.html', 'info_dict': { 'id': '72371d3c-d998-49f3-8860-d168cfdf4966', 'title': "Alice nel paese delle meraviglie", - 'description': "di Lewis Carrol letto da Aldo Busi", - }, - 'playlist_count': 11, - } + 'description': "di Lewis Carrol letto da Aldo Busi"}, + 'playlist_count': 11} def _real_extract(self, url): playlist_id = self._match_id(url) playlist_webpage = self._download_webpage(url, playlist_id) - playlist_title = unescapeHTML(self._html_search_regex(r'data-playlist-title="(.+?)"', playlist_webpage, 'title')) - playlist_creator = self._html_search_meta('nomeProgramma', playlist_webpage) - playlist_description = get_element_by_class('textDescriptionProgramma', playlist_webpage) + playlist_title = unescapeHTML(self._html_search_regex( + r'data-playlist-title="(.+?)"', playlist_webpage, 'title')) + playlist_creator = self._html_search_meta( + 'nomeProgramma', playlist_webpage) + playlist_description = get_element_by_class( + 'textDescriptionProgramma', playlist_webpage) - player_href = self._html_search_regex(r'data-player-href="(.+?)"', playlist_webpage, 'href') + player_href = self._html_search_regex( + r'data-player-href="(.+?)"', playlist_webpage, 'href') list_url = urljoin(url, player_href) entries = self.get_playlist(list_url) @@ -595,7 +603,7 @@ class RaiPlayRadioPlaylistIE(RaiPlayRadioBaseIE): 'track': entry['title'], 'track_number': index, 'album': playlist_title, - 'artist': playlist_creator, - }) + 'artist': playlist_creator}) - return self.playlist_result(entries, playlist_id, playlist_title, playlist_description) + return self.playlist_result( + entries, playlist_id, playlist_title, playlist_description) From e918882d29d033f8a273a326f77381b9786a77c0 Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Tue, 17 Mar 2020 23:54:22 +0100 Subject: [PATCH 03/10] docs/supportedsites.md is automatically generated --- docs/supportedsites.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index afd34f785..9ae6e5c96 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -718,8 +718,6 @@ - **RaiPlay** - **RaiPlayLive** - **RaiPlayPlaylist** - - **RaiPlayRadio** - - **RaiPlayRadioPlaylist** - **RayWenderlich** - **RayWenderlichCourse** - **RBMARadio** From a58aafd664731053475de5d1d42381f2c70c30f5 Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Tue, 17 Mar 2020 23:56:14 +0100 Subject: [PATCH 04/10] Use a uid instead of a URL --- youtube_dl/extractor/rai.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py index a758d5653..a594385f5 100644 --- a/youtube_dl/extractor/rai.py +++ b/youtube_dl/extractor/rai.py @@ -528,8 +528,8 @@ class RaiPlayRadioBaseIE(InfoExtractor): parser.close() return parser.items - def get_playlist_iter(self, url): - webpage = self._download_webpage(url, url) + def get_playlist_iter(self, url, uid): + webpage = self._download_webpage(url, uid) for attrs in self.parse_list(webpage): title = attrs['data-title'].strip() webpage = urljoin(url, attrs['data-mediapolis']) @@ -564,9 +564,10 @@ class RaiPlayRadioIE(RaiPlayRadioBaseIE): 'language': 'it'}} def _real_extract(self, url): + audio_id = self._match_id(url) list_url = url.replace('.html', '-list.html') - for entry in self.get_playlist_iter(list_url): - if entry['id'] == self._match_id(url): + for entry in self.get_playlist_iter(list_url, audio_id): + if entry['id'] == audio_id: return entry @@ -597,7 +598,7 @@ class RaiPlayRadioPlaylistIE(RaiPlayRadioBaseIE): r'data-player-href="(.+?)"', playlist_webpage, 'href') list_url = urljoin(url, player_href) - entries = self.get_playlist(list_url) + entries = self.get_playlist(list_url, playlist_id) for index, entry in enumerate(entries, start=1): entry.update({ 'track': entry['title'], From 603bce83e1787e696d70d3b5efcbb709af28312b Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Tue, 17 Mar 2020 23:57:34 +0100 Subject: [PATCH 05/10] Avoid guessing filetype --- youtube_dl/extractor/rai.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py index a594385f5..915b0e1ae 100644 --- a/youtube_dl/extractor/rai.py +++ b/youtube_dl/extractor/rai.py @@ -532,14 +532,12 @@ class RaiPlayRadioBaseIE(InfoExtractor): webpage = self._download_webpage(url, uid) for attrs in self.parse_list(webpage): title = attrs['data-title'].strip() - webpage = urljoin(url, attrs['data-mediapolis']) - audio_url = compat_str(self._request_webpage( - webpage, title).geturl()) + audio_url = urljoin(url, attrs['data-mediapolis']) yield { 'url': audio_url, 'id': attrs['data-uniquename'].lstrip('ContentItem-'), 'title': title, - 'ext': determine_ext(audio_url), + 'ext': 'mp3', 'thumbnail': urljoin(url, attrs['data-image']), 'language': 'it'} From 686f8f60bac675f36e43f97228605a2dd7620236 Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Tue, 17 Mar 2020 23:59:46 +0100 Subject: [PATCH 06/10] playlist_title is optional --- youtube_dl/extractor/rai.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py index 915b0e1ae..ad51bf123 100644 --- a/youtube_dl/extractor/rai.py +++ b/youtube_dl/extractor/rai.py @@ -601,8 +601,10 @@ class RaiPlayRadioPlaylistIE(RaiPlayRadioBaseIE): entry.update({ 'track': entry['title'], 'track_number': index, - 'album': playlist_title, - 'artist': playlist_creator}) + 'artist': playlist_creator, + }) + if playlist_title: + entry['album'] = playlist_title return self.playlist_result( entries, playlist_id, playlist_title, playlist_description) From 0a21514afc821e15c97566a39028bfb123545e92 Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Wed, 18 Mar 2020 00:00:04 +0100 Subject: [PATCH 07/10] 80 character is a soft limit --- youtube_dl/extractor/rai.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py index ad51bf123..284260d78 100644 --- a/youtube_dl/extractor/rai.py +++ b/youtube_dl/extractor/rai.py @@ -549,17 +549,15 @@ class RaiPlayRadioIE(RaiPlayRadioBaseIE): _VALID_URL = r'%s/audio/.+?-(?P%s)\.html' % ( RaiPlayRadioBaseIE._BASE, RaiBaseIE._UUID_RE) _TEST = { - 'url': 'https://www.raiplayradio.it/audio/2019/07/' - 'RADIO3---LEZIONI-DI-MUSICA-' - '36b099ff-4123-4443-9bf9-38e43ef5e025.html', + 'url': 'https://www.raiplayradio.it/audio/2019/07/RADIO3---LEZIONI-DI-MUSICA-36b099ff-4123-4443-9bf9-38e43ef5e025.html', 'info_dict': { 'id': '36b099ff-4123-4443-9bf9-38e43ef5e025', 'ext': 'mp3', 'title': 'Dal "Chiaro di luna" al "Clair de lune", ' 'prima parte con Giovanni Bietti', - 'thumbnail': 'https://www.raiplayradio.it/cropgd/400x400/dl/' - 'img/2019/07/08/1562590329054_luna.jpg', - 'language': 'it'}} + 'thumbnail': 'https://www.raiplayradio.it/cropgd/400x400/dl/img/2019/07/08/1562590329054_luna.jpg', + 'language': 'it', + }} def _real_extract(self, url): audio_id = self._match_id(url) @@ -573,14 +571,14 @@ class RaiPlayRadioPlaylistIE(RaiPlayRadioBaseIE): _VALID_URL = r'%s/playlist/.+?-(?P%s)\.html' % ( RaiPlayRadioBaseIE._BASE, RaiBaseIE._UUID_RE) _TEST = { - 'url': 'https://www.raiplayradio.it/playlist/2017/12/' - 'Alice-nel-paese-delle-meraviglie-' - '72371d3c-d998-49f3-8860-d168cfdf4966.html', + 'url': 'https://www.raiplayradio.it/playlist/2017/12/Alice-nel-paese-delle-meraviglie-72371d3c-d998-49f3-8860-d168cfdf4966.html', 'info_dict': { 'id': '72371d3c-d998-49f3-8860-d168cfdf4966', 'title': "Alice nel paese delle meraviglie", - 'description': "di Lewis Carrol letto da Aldo Busi"}, - 'playlist_count': 11} + 'description': "di Lewis Carrol letto da Aldo Busi", + }, + 'playlist_count': 11, + } def _real_extract(self, url): playlist_id = self._match_id(url) From 79f4944f585736ef1fb584e2c92d8f19a552c494 Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Wed, 18 Mar 2020 00:02:14 +0100 Subject: [PATCH 08/10] Avoid breaking if thumbnail is missing --- youtube_dl/extractor/rai.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py index 284260d78..ef1ba87a8 100644 --- a/youtube_dl/extractor/rai.py +++ b/youtube_dl/extractor/rai.py @@ -533,13 +533,16 @@ class RaiPlayRadioBaseIE(InfoExtractor): for attrs in self.parse_list(webpage): title = attrs['data-title'].strip() audio_url = urljoin(url, attrs['data-mediapolis']) - yield { + entry = { 'url': audio_url, 'id': attrs['data-uniquename'].lstrip('ContentItem-'), 'title': title, 'ext': 'mp3', - 'thumbnail': urljoin(url, attrs['data-image']), - 'language': 'it'} + 'language': 'it', + } + if 'data-image' in attrs: + entry['thumbnail'] = urljoin(url, attrs['data-image']) + yield entry def get_playlist(self, *args, **kwargs): return list(self.get_playlist_iter(*args, **kwargs)) From 78852c2dd1f6fbba3521f845e94b4d4f60cfabd0 Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Wed, 18 Mar 2020 00:09:52 +0100 Subject: [PATCH 09/10] Avoid using direct URL for the thumbnail --- youtube_dl/extractor/rai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py index ef1ba87a8..c169ff811 100644 --- a/youtube_dl/extractor/rai.py +++ b/youtube_dl/extractor/rai.py @@ -558,7 +558,7 @@ class RaiPlayRadioIE(RaiPlayRadioBaseIE): 'ext': 'mp3', 'title': 'Dal "Chiaro di luna" al "Clair de lune", ' 'prima parte con Giovanni Bietti', - 'thumbnail': 'https://www.raiplayradio.it/cropgd/400x400/dl/img/2019/07/08/1562590329054_luna.jpg', + 'thumbnail': r're:^https?://.*\.jpg$', 'language': 'it', }} From 545cffa63aef49b3ad094f01c57a721ca0b8834e Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Wed, 18 Mar 2020 08:41:36 +0100 Subject: [PATCH 10/10] Flake8 fix --- youtube_dl/extractor/rai.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py index c169ff811..149ae0a81 100644 --- a/youtube_dl/extractor/rai.py +++ b/youtube_dl/extractor/rai.py @@ -560,7 +560,8 @@ class RaiPlayRadioIE(RaiPlayRadioBaseIE): 'prima parte con Giovanni Bietti', 'thumbnail': r're:^https?://.*\.jpg$', 'language': 'it', - }} + } + } def _real_extract(self, url): audio_id = self._match_id(url)