From c66cc3cc5bebfe94b8218e1bf1ca5ecffb063fa1 Mon Sep 17 00:00:00 2001 From: Pete Johns Date: Sun, 12 May 2019 13:21:42 +1000 Subject: [PATCH 1/8] Handle SBS News URLs without IDs --- youtube_dl/extractor/sbs.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index 0e623ff7b..9dd964cec 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -1,6 +1,8 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .common import InfoExtractor from ..utils import ( smuggle_url, @@ -10,15 +12,29 @@ from ..utils import ( class SBSIE(InfoExtractor): IE_DESC = 'sbs.com.au' - _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/video/(?:single/)?(?P[0-9]+)' + _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/(?:video/)?(?:single/)?([0-9]+|[0-9a-z-]+)' _TESTS = [{ + 'url': 'https://www.sbs.com.au/news/are-the-campaigns-working-voters-speak-out', + 'md5': '2b73ddcbb597f24a87167826c47398f8', + 'info_dict': { + 'id': 'Vznr2YGb83mF', + 'ext': 'mp4', + 'title': 'Are the campaigns cutting through?', + 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e', + 'thumbnail': r're:http://.*\.jpg', + 'duration': 146, + 'timestamp': 1557552900, + 'upload_date': '20190511', + 'uploader': 'SBSC', + } + }, { # Original URL is handled by the generic IE which finds the iframe: # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation 'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed', 'md5': '3150cf278965eeabb5b4cea1c963fe0a', 'info_dict': { - 'id': '320403011771', + 'id': '_rFBPRPO4pMR', 'ext': 'mp4', 'title': 'Dingo Conservation (The Feed)', 'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5', @@ -36,8 +52,21 @@ class SBSIE(InfoExtractor): 'only_matching': True, }] + def video_id_from_page_contents(self, url): + page_contents = self._download_webpage(url, None) + video_id = self._search_regex(r'id="video-(\d+)"', page_contents, 'video id') + return video_id + + def video_id(self, url): + ID_BEARING_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/video/(?:single/)?(?P[0-9]+)' + match = re.match(ID_BEARING_URL, url) + if match: + return match.group('id') + else: + return self.video_id_from_page_contents(url) + def _real_extract(self, url): - video_id = self._match_id(url) + video_id = self.video_id(url) player_params = self._download_json( 'http://www.sbs.com.au/api/video_pdkvars/id/%s?form=json' % video_id, video_id) From 744df7ab325c6ee28c5a877002adfba91f0e7a36 Mon Sep 17 00:00:00 2001 From: Pete Johns Date: Sun, 12 May 2019 20:42:56 +1000 Subject: [PATCH 2/8] Extract SBSNewsIE from SBSIE --- youtube_dl/extractor/extractors.py | 5 ++- youtube_dl/extractor/sbs.py | 65 ++++++++++++++++-------------- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 3037b5a45..7c3cc6a14 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1005,7 +1005,10 @@ from .safari import ( ) from .sapo import SapoIE from .savefrom import SaveFromIE -from .sbs import SBSIE +from .sbs import ( + SBSIE, + SBSNewsIE, +) from .screencast import ScreencastIE from .screencastomatic import ScreencastOMaticIE from .scrippsnetworks import ScrippsNetworksWatchIE diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index 9dd964cec..07a0c33fa 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -1,8 +1,6 @@ # coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor from ..utils import ( smuggle_url, @@ -12,23 +10,9 @@ from ..utils import ( class SBSIE(InfoExtractor): IE_DESC = 'sbs.com.au' - _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/(?:video/)?(?:single/)?([0-9]+|[0-9a-z-]+)' + _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/video/(?:single/)?(?P[0-9]+)' _TESTS = [{ - 'url': 'https://www.sbs.com.au/news/are-the-campaigns-working-voters-speak-out', - 'md5': '2b73ddcbb597f24a87167826c47398f8', - 'info_dict': { - 'id': 'Vznr2YGb83mF', - 'ext': 'mp4', - 'title': 'Are the campaigns cutting through?', - 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e', - 'thumbnail': r're:http://.*\.jpg', - 'duration': 146, - 'timestamp': 1557552900, - 'upload_date': '20190511', - 'uploader': 'SBSC', - } - }, { # Original URL is handled by the generic IE which finds the iframe: # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation 'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed', @@ -52,21 +36,8 @@ class SBSIE(InfoExtractor): 'only_matching': True, }] - def video_id_from_page_contents(self, url): - page_contents = self._download_webpage(url, None) - video_id = self._search_regex(r'id="video-(\d+)"', page_contents, 'video id') - return video_id - - def video_id(self, url): - ID_BEARING_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/video/(?:single/)?(?P[0-9]+)' - match = re.match(ID_BEARING_URL, url) - if match: - return match.group('id') - else: - return self.video_id_from_page_contents(url) - def _real_extract(self, url): - video_id = self.video_id(url) + video_id = self._video_id(url) player_params = self._download_json( 'http://www.sbs.com.au/api/video_pdkvars/id/%s?form=json' % video_id, video_id) @@ -93,3 +64,35 @@ class SBSIE(InfoExtractor): 'id': video_id, 'url': smuggle_url(self._proto_relative_url(theplatform_url), {'force_smil_url': True}), } + + def _video_id(self, url): + return self._match_id(url) + + +class SBSNewsIE(SBSIE): + _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/news/(?P[0-9a-z-]+)' + + _TESTS = [{ + 'url': 'https://www.sbs.com.au/news/are-the-campaigns-working-voters-speak-out', + 'only_matching': True, + 'md5': '2b73ddcbb597f24a87167826c47398f8', + 'info_dict': { + 'id': 'Vznr2YGb83mF', + 'ext': 'mp4', + 'title': 'Are the campaigns cutting through?', + 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e', + 'thumbnail': r're:http://.*\.jpg', + 'duration': 146, + 'timestamp': 1557552900, + 'upload_date': '20190511', + 'uploader': 'SBSC', + }, + }, { + 'url': 'https://www.sbs.com.au/news/sbs-world-news-bulletin-may-11', + 'only_matching': True, + }] + + def _video_id(self, url): + slug = self._match_id(url) + page_contents = self._download_webpage(url, slug) + return self._search_regex(r'id="video-(\d+)"', page_contents, 'video id') From 75256313a60c74bcfb55d6cfbdadcb8235ec1a26 Mon Sep 17 00:00:00 2001 From: Pete Johns Date: Mon, 13 May 2019 07:45:03 +1000 Subject: [PATCH 3/8] Anchor re to prevent false-positives --- youtube_dl/extractor/sbs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index 07a0c33fa..92dbe2f51 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -70,7 +70,7 @@ class SBSIE(InfoExtractor): class SBSNewsIE(SBSIE): - _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/news/(?P[0-9a-z-]+)' + _VALID_URL = r'^https?://(?:www\.)?sbs\.com\.au/news/(?P[0-9a-z-]+)$' _TESTS = [{ 'url': 'https://www.sbs.com.au/news/are-the-campaigns-working-voters-speak-out', From 5e04505a966a77f3896ac181b3121f83cc9af8cd Mon Sep 17 00:00:00 2001 From: Pete Johns Date: Mon, 13 Apr 2020 13:14:02 +1000 Subject: [PATCH 4/8] Remove data obviated by 'only_matching' flag --- youtube_dl/extractor/sbs.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index 92dbe2f51..230e65414 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -75,18 +75,6 @@ class SBSNewsIE(SBSIE): _TESTS = [{ 'url': 'https://www.sbs.com.au/news/are-the-campaigns-working-voters-speak-out', 'only_matching': True, - 'md5': '2b73ddcbb597f24a87167826c47398f8', - 'info_dict': { - 'id': 'Vznr2YGb83mF', - 'ext': 'mp4', - 'title': 'Are the campaigns cutting through?', - 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e', - 'thumbnail': r're:http://.*\.jpg', - 'duration': 146, - 'timestamp': 1557552900, - 'upload_date': '20190511', - 'uploader': 'SBSC', - }, }, { 'url': 'https://www.sbs.com.au/news/sbs-world-news-bulletin-may-11', 'only_matching': True, From fd90cab05a2d47c56c38eef99b97760b8eb5509a Mon Sep 17 00:00:00 2001 From: Pete Johns Date: Mon, 13 Apr 2020 13:23:08 +1000 Subject: [PATCH 5/8] _VALID_URL regular expressions are not anchored --- youtube_dl/extractor/sbs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index 230e65414..2e639efc1 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -70,7 +70,7 @@ class SBSIE(InfoExtractor): class SBSNewsIE(SBSIE): - _VALID_URL = r'^https?://(?:www\.)?sbs\.com\.au/news/(?P[0-9a-z-]+)$' + _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/news/(?P[0-9a-z-]+)' _TESTS = [{ 'url': 'https://www.sbs.com.au/news/are-the-campaigns-working-voters-speak-out', From 685ed27a4f11d600a2281e09884eec50fbfb3772 Mon Sep 17 00:00:00 2001 From: Pete Johns Date: Mon, 13 Apr 2020 13:26:23 +1000 Subject: [PATCH 6/8] Use suitable() to determine which InfoExtractor class to use --- youtube_dl/extractor/sbs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index 2e639efc1..2fc56e0f9 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -80,6 +80,10 @@ class SBSNewsIE(SBSIE): 'only_matching': True, }] + @classmethod + def suitable(cls, url): + return False if SBSIE.suitable(url) else super(SBSNewsIE, cls).suitable(url) + def _video_id(self, url): slug = self._match_id(url) page_contents = self._download_webpage(url, slug) From a6376446ec7b181ed676427a66523ce925c6ec46 Mon Sep 17 00:00:00 2001 From: Pete Johns Date: Mon, 13 Apr 2020 13:35:27 +1000 Subject: [PATCH 7/8] Don't inherit from SBSIE It's unclear from https://github.com/ytdl-org/youtube-dl/pull/21066#discussion_r407193002 why this is the case as the codebase has many examples of other InfoExtractor classes inheriting from derived classes. --- youtube_dl/extractor/sbs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index 2fc56e0f9..d9ad9d16e 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -69,7 +69,7 @@ class SBSIE(InfoExtractor): return self._match_id(url) -class SBSNewsIE(SBSIE): +class SBSNewsIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/news/(?P[0-9a-z-]+)' _TESTS = [{ From 5d6e557d0f87978351e4a5ee4dfb73792f5888b2 Mon Sep 17 00:00:00 2001 From: Pete Johns Date: Mon, 13 Apr 2020 15:04:27 +1000 Subject: [PATCH 8/8] Extract SBSBaseIE with common _real_extract(), which follows established BaseIE pattern --- youtube_dl/extractor/sbs.py | 66 ++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index d9ad9d16e..d8ba43d38 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -8,33 +8,9 @@ from ..utils import ( ) -class SBSIE(InfoExtractor): - IE_DESC = 'sbs.com.au' - _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/video/(?:single/)?(?P[0-9]+)' - - _TESTS = [{ - # Original URL is handled by the generic IE which finds the iframe: - # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation - 'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed', - 'md5': '3150cf278965eeabb5b4cea1c963fe0a', - 'info_dict': { - 'id': '_rFBPRPO4pMR', - 'ext': 'mp4', - 'title': 'Dingo Conservation (The Feed)', - 'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5', - 'thumbnail': r're:http://.*\.jpg', - 'duration': 308, - 'timestamp': 1408613220, - 'upload_date': '20140821', - 'uploader': 'SBSC', - }, - }, { - 'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed', - 'only_matching': True, - }, { - 'url': 'http://www.sbs.com.au/news/video/471395907773/The-Feed-July-9', - 'only_matching': True, - }] +class SBSBaseIE(InfoExtractor): + def _video_id(self, url): + raise NotImplementedError('SBS InfoExtractor classes must implement _video_id()') def _real_extract(self, url): video_id = self._video_id(url) @@ -65,18 +41,48 @@ class SBSIE(InfoExtractor): 'url': smuggle_url(self._proto_relative_url(theplatform_url), {'force_smil_url': True}), } + +class SBSIE(SBSBaseIE): + IE_DESC = 'sbs.com.au' + _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/video/(?:single/)?(?P[0-9]+)' + + _TESTS = [{ + # Original URL is handled by the generic IE which finds the iframe: + # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation + 'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed', + 'md5': '3150cf278965eeabb5b4cea1c963fe0a', + 'info_dict': { + 'id': '_rFBPRPO4pMR', + 'ext': 'mp4', + 'title': 'Dingo Conservation (The Feed)', + 'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5', + 'thumbnail': r're:http://.*\.jpg', + 'duration': 308, + 'timestamp': 1408613220, + 'upload_date': '20140821', + 'uploader': 'SBSC', + }, + }, { + 'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed', + 'only_matching': True, + }, { + 'url': 'http://www.sbs.com.au/news/video/471395907773/The-Feed-July-9', + 'only_matching': True, + }] + def _video_id(self, url): return self._match_id(url) -class SBSNewsIE(InfoExtractor): +class SBSNewsIE(SBSBaseIE): + IE_DESC = 'sbs.com.au:news' _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/news/(?P[0-9a-z-]+)' _TESTS = [{ - 'url': 'https://www.sbs.com.au/news/are-the-campaigns-working-voters-speak-out', + 'url': 'https://www.sbs.com.au/news/rio-s-christ-the-redeemer-dons-doctor-s-coat-to-honour-coronavirus-medics', 'only_matching': True, }, { - 'url': 'https://www.sbs.com.au/news/sbs-world-news-bulletin-may-11', + 'url': 'https://www.sbs.com.au/news/catch-up-sbs-world-news-11-april-2020', 'only_matching': True, }]