From 8d1380697bef17901d60f954f373299f1df90dba Mon Sep 17 00:00:00 2001 From: Roxedus Date: Fri, 27 Dec 2019 13:27:35 +0100 Subject: [PATCH 1/9] [nrk] fix and update extractor Added new error message Fixed tests, some of the existing ones were removed from the site --- youtube_dl/extractor/nrk.py | 52 ++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py index 60933f069..848e2a949 100644 --- a/youtube_dl/extractor/nrk.py +++ b/youtube_dl/extractor/nrk.py @@ -105,6 +105,7 @@ class NRKBaseIE(InfoExtractor): MESSAGES = { 'ProgramRightsAreNotReady': 'Du kan dessverre ikke se eller høre programmet', 'ProgramRightsHasExpired': 'Programmet har gått ut', + 'NoProgramRights': 'Ikke tilgjengelig', 'ProgramIsGeoBlocked': 'NRK har ikke rettigheter til å vise dette programmet utenfor Norge', } message_type = data.get('messageType', '') @@ -255,6 +256,19 @@ class NRKTVIE(NRKBaseIE): ''' % _EPISODE_RE _API_HOSTS = ('psapi-ne.nrk.no', 'psapi-we.nrk.no') _TESTS = [{ + 'url': 'https://tv.nrk.no/serie/blank/sesong/1/episode/3', + 'md5': '22053d8a7641709940ad195853b197f0', + 'info_dict': { + 'id': 'MUHH69000318AA', + 'ext': 'mp4', + 'title': 'Uten mål og mening 3:9', + 'description': 'md5:32692999accf028f406bf3bc73eb29ac', + 'duration': 1698, + 'series': 'Blank', + 'episode': '3:9', + 'age_limit': 12 + }, + }, { 'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014', 'md5': '9a167e54d04671eb6317a37b7bc8a280', 'info_dict': { @@ -370,7 +384,27 @@ class NRKTVIE(NRKBaseIE): class NRKTVEpisodeIE(InfoExtractor): _VALID_URL = r'https?://tv\.nrk\.no/serie/(?P[^/]+/sesong/\d+/episode/\d+)' - _TEST = { + _TESTS = [{ + 'url': 'https://tv.nrk.no/serie/hellums-kro/sesong/1/episode/2', + 'info_dict': { + 'id': 'MUHH36005220BA', + 'ext': 'mp4', + 'title': 'Kro, krig og kjærlighet 2:6', + 'description': 'Jan tviholder på sine prinsipper. ' + 'Oddbjørn er skeptisk til Jans evne til å lede ' + 'dem og bestemmer seg for å ta saken i egne hender.', + 'duration': 1563, + 'series': 'Hellums kro', + 'season_number': 1, + 'episode_number': 2, + 'episode': '2:6', + 'age_limit': 6 + }, + 'params': { + 'skip_download': True, + }, + }, { + # Expired show 'url': 'https://tv.nrk.no/serie/backstage/sesong/1/episode/8', 'info_dict': { 'id': 'MSUI14000816AA', @@ -386,7 +420,7 @@ class NRKTVEpisodeIE(InfoExtractor): 'params': { 'skip_download': True, }, - } + }] def _real_extract(self, url): display_id = self._match_id(url) @@ -408,7 +442,7 @@ class NRKTVSerieBaseIE(InfoExtractor): self._search_regex( (r'INITIAL_DATA(?:_V\d)?_*\s*=\s*({.+?})\s*;', r'({.+?})\s*,\s*"[^"]+"\s*\)\s*'), - webpage, 'config', default='{}' if not fatal else NO_DEFAULT), + webpage.replace("undefined", '"undefined"'), 'config', default='{}' if not fatal else NO_DEFAULT), display_id, fatal=False) if not config: return @@ -479,6 +513,16 @@ class NRKTVSeriesIE(NRKTVSerieBaseIE): _VALID_URL = r'https?://(?:tv|radio)\.nrk(?:super)?\.no/serie/(?P[^/]+)' _ITEM_RE = r'(?:data-season=["\']|id=["\']season-)(?P\d+)' _TESTS = [{ + 'url': 'https://tv.nrk.no/serie/blank', + 'info_dict': { + 'id': 'blank', + 'title': 'Blank', + 'description': 'Da Markus dro fra bygda var han den kuleste fyren med den fineste dama. ' + 'Etter kort tid er han tilbake. Og ingenting er som før. Norsk dramaserie.', + 'age_limit': None + }, + 'playlist_mincount': 30, + }, { # new layout, seasons 'url': 'https://tv.nrk.no/serie/backstage', 'info_dict': { @@ -648,7 +692,7 @@ class NRKSkoleIE(InfoExtractor): _TESTS = [{ 'url': 'https://www.nrk.no/skole/?page=search&q=&mediaId=14099', - 'md5': '6bc936b01f9dd8ed45bc58b252b2d9b6', + 'md5': '18c12c3d071953c3bf8d54ef6b2587b7', 'info_dict': { 'id': '6021', 'ext': 'mp4', From fa9516583db4eeadf17acc99e139bc55e1959e85 Mon Sep 17 00:00:00 2001 From: Roxedus Date: Mon, 30 Dec 2019 03:16:16 +0100 Subject: [PATCH 2/9] Update tests --- youtube_dl/extractor/nrk.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py index 848e2a949..4a1424eb7 100644 --- a/youtube_dl/extractor/nrk.py +++ b/youtube_dl/extractor/nrk.py @@ -256,17 +256,15 @@ class NRKTVIE(NRKBaseIE): ''' % _EPISODE_RE _API_HOSTS = ('psapi-ne.nrk.no', 'psapi-we.nrk.no') _TESTS = [{ - 'url': 'https://tv.nrk.no/serie/blank/sesong/1/episode/3', - 'md5': '22053d8a7641709940ad195853b197f0', + 'url': 'https://tv.nrk.no/program/MDDP12000117', + 'md5': '8270824df46ec629b66aeaa5796b36fb', 'info_dict': { - 'id': 'MUHH69000318AA', + 'id': 'MDDP12000117AA', 'ext': 'mp4', - 'title': 'Uten mål og mening 3:9', - 'description': 'md5:32692999accf028f406bf3bc73eb29ac', - 'duration': 1698, - 'series': 'Blank', - 'episode': '3:9', - 'age_limit': 12 + 'title': 'Alarm Trolltunga', + 'description': 'md5:46923a6e6510eefcce23d5ef2a58f2ce', + 'duration': 2223, + 'age_limit': 6 }, }, { 'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014', From 087607cb786c5075b4886eebdc10dee9a82aa096 Mon Sep 17 00:00:00 2001 From: Roxedus Date: Fri, 3 Jan 2020 13:58:54 +0100 Subject: [PATCH 3/9] coding conventions comply to coding conventions --- youtube_dl/extractor/nrk.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py index 4a1424eb7..5578b858d 100644 --- a/youtube_dl/extractor/nrk.py +++ b/youtube_dl/extractor/nrk.py @@ -16,6 +16,7 @@ from ..utils import ( parse_age_limit, parse_duration, try_get, + js_to_json, ) @@ -437,10 +438,11 @@ class NRKTVEpisodeIE(InfoExtractor): class NRKTVSerieBaseIE(InfoExtractor): def _extract_series(self, webpage, display_id, fatal=True): config = self._parse_json( - self._search_regex( - (r'INITIAL_DATA(?:_V\d)?_*\s*=\s*({.+?})\s*;', - r'({.+?})\s*,\s*"[^"]+"\s*\)\s*'), - webpage.replace("undefined", '"undefined"'), 'config', default='{}' if not fatal else NO_DEFAULT), + js_to_json( + self._search_regex( + (r'INITIAL_DATA(?:_V\d)?_*\s*=\s*({.+?})\s*;', + r'({.+?})\s*,\s*"[^"]+"\s*\)\s*'), + webpage, 'config', default='{}' if not fatal else NO_DEFAULT)), display_id, fatal=False) if not config: return @@ -515,8 +517,7 @@ class NRKTVSeriesIE(NRKTVSerieBaseIE): 'info_dict': { 'id': 'blank', 'title': 'Blank', - 'description': 'Da Markus dro fra bygda var han den kuleste fyren med den fineste dama. ' - 'Etter kort tid er han tilbake. Og ingenting er som før. Norsk dramaserie.', + 'description': 'Da Markus dro fra bygda var han den kuleste fyren med den fineste dama. Etter kort tid er han tilbake. Og ingenting er som før. Norsk dramaserie.', 'age_limit': None }, 'playlist_mincount': 30, From aa31a2c2356bfd95972136014cb24b552aa5b8f0 Mon Sep 17 00:00:00 2001 From: Roxedus Date: Sun, 5 Jan 2020 11:52:10 +0100 Subject: [PATCH 4/9] Update --- youtube_dl/extractor/nrk.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py index 5578b858d..1379ac7bf 100644 --- a/youtube_dl/extractor/nrk.py +++ b/youtube_dl/extractor/nrk.py @@ -11,12 +11,12 @@ from ..compat import ( from ..utils import ( ExtractorError, int_or_none, + js_to_json, JSON_LD_RE, NO_DEFAULT, parse_age_limit, parse_duration, try_get, - js_to_json, ) @@ -389,9 +389,7 @@ class NRKTVEpisodeIE(InfoExtractor): 'id': 'MUHH36005220BA', 'ext': 'mp4', 'title': 'Kro, krig og kjærlighet 2:6', - 'description': 'Jan tviholder på sine prinsipper. ' - 'Oddbjørn er skeptisk til Jans evne til å lede ' - 'dem og bestemmer seg for å ta saken i egne hender.', + 'description': 'Jan tviholder på sine prinsipper. Oddbjørn er skeptisk til Jans evne til å lede dem og bestemmer seg for å ta saken i egne hender.', 'duration': 1563, 'series': 'Hellums kro', 'season_number': 1, @@ -518,7 +516,6 @@ class NRKTVSeriesIE(NRKTVSerieBaseIE): 'id': 'blank', 'title': 'Blank', 'description': 'Da Markus dro fra bygda var han den kuleste fyren med den fineste dama. Etter kort tid er han tilbake. Og ingenting er som før. Norsk dramaserie.', - 'age_limit': None }, 'playlist_mincount': 30, }, { From 10f2af0bb29d228ebbb7098a52c815c8c97ef272 Mon Sep 17 00:00:00 2001 From: Roxedus Date: Sun, 5 Jan 2020 22:50:52 +0100 Subject: [PATCH 5/9] Updates --- youtube_dl/extractor/nrk.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py index 1379ac7bf..ca3466c69 100644 --- a/youtube_dl/extractor/nrk.py +++ b/youtube_dl/extractor/nrk.py @@ -389,7 +389,7 @@ class NRKTVEpisodeIE(InfoExtractor): 'id': 'MUHH36005220BA', 'ext': 'mp4', 'title': 'Kro, krig og kjærlighet 2:6', - 'description': 'Jan tviholder på sine prinsipper. Oddbjørn er skeptisk til Jans evne til å lede dem og bestemmer seg for å ta saken i egne hender.', + 'description': 'md5:b32a7dc0b1ed27c8064f58b97bda4350', 'duration': 1563, 'series': 'Hellums kro', 'season_number': 1, @@ -436,12 +436,11 @@ class NRKTVEpisodeIE(InfoExtractor): class NRKTVSerieBaseIE(InfoExtractor): def _extract_series(self, webpage, display_id, fatal=True): config = self._parse_json( - js_to_json( - self._search_regex( - (r'INITIAL_DATA(?:_V\d)?_*\s*=\s*({.+?})\s*;', - r'({.+?})\s*,\s*"[^"]+"\s*\)\s*'), - webpage, 'config', default='{}' if not fatal else NO_DEFAULT)), - display_id, fatal=False) + self._search_regex( + (r'INITIAL_DATA(?:_V\d)?_*\s*=\s*({.+?})\s*;', + r'({.+?})\s*,\s*"[^"]+"\s*\)\s*'), + webpage, 'config', default='{}' if not fatal else NO_DEFAULT), + display_id, fatal=False, transform_source=js_to_json) if not config: return return try_get( @@ -515,7 +514,7 @@ class NRKTVSeriesIE(NRKTVSerieBaseIE): 'info_dict': { 'id': 'blank', 'title': 'Blank', - 'description': 'Da Markus dro fra bygda var han den kuleste fyren med den fineste dama. Etter kort tid er han tilbake. Og ingenting er som før. Norsk dramaserie.', + 'description': 'md5:7664b4e7e77dc6810cd3bca367c25b6e', }, 'playlist_mincount': 30, }, { From b72c20fbcafe5c1f393c9c94e44e652d926ad7fb Mon Sep 17 00:00:00 2001 From: Sergey M Date: Mon, 6 Jan 2020 06:23:53 +0700 Subject: [PATCH 6/9] Update nrk.py --- youtube_dl/extractor/nrk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py index ca3466c69..7fac3059f 100644 --- a/youtube_dl/extractor/nrk.py +++ b/youtube_dl/extractor/nrk.py @@ -268,6 +268,7 @@ class NRKTVIE(NRKBaseIE): 'age_limit': 6 }, }, { + # NoProgramRights 'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014', 'md5': '9a167e54d04671eb6317a37b7bc8a280', 'info_dict': { @@ -279,6 +280,7 @@ class NRKTVIE(NRKBaseIE): 'series': '20 spørsmål', 'episode': '23.05.2014', }, + 'only_matching': True, }, { 'url': 'https://tv.nrk.no/program/mdfp15000514', 'info_dict': { From b74b7a2a7045e9df3843f662719fe9d1f0b37db4 Mon Sep 17 00:00:00 2001 From: Sergey M Date: Mon, 6 Jan 2020 06:25:00 +0700 Subject: [PATCH 7/9] Update nrk.py --- youtube_dl/extractor/nrk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py index 7fac3059f..b1e86a3ff 100644 --- a/youtube_dl/extractor/nrk.py +++ b/youtube_dl/extractor/nrk.py @@ -11,8 +11,8 @@ from ..compat import ( from ..utils import ( ExtractorError, int_or_none, - js_to_json, JSON_LD_RE, + js_to_json, NO_DEFAULT, parse_age_limit, parse_duration, From d6027ceee16f2ce3bb5c364512505794c65a0b2b Mon Sep 17 00:00:00 2001 From: Sergey M Date: Mon, 6 Jan 2020 06:28:13 +0700 Subject: [PATCH 8/9] Update nrk.py --- youtube_dl/extractor/nrk.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py index b1e86a3ff..1d6f42e86 100644 --- a/youtube_dl/extractor/nrk.py +++ b/youtube_dl/extractor/nrk.py @@ -268,7 +268,6 @@ class NRKTVIE(NRKBaseIE): 'age_limit': 6 }, }, { - # NoProgramRights 'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014', 'md5': '9a167e54d04671eb6317a37b7bc8a280', 'info_dict': { @@ -280,7 +279,7 @@ class NRKTVIE(NRKBaseIE): 'series': '20 spørsmål', 'episode': '23.05.2014', }, - 'only_matching': True, + 'skip': 'NoProgramRights', }, { 'url': 'https://tv.nrk.no/program/mdfp15000514', 'info_dict': { @@ -403,7 +402,6 @@ class NRKTVEpisodeIE(InfoExtractor): 'skip_download': True, }, }, { - # Expired show 'url': 'https://tv.nrk.no/serie/backstage/sesong/1/episode/8', 'info_dict': { 'id': 'MSUI14000816AA', @@ -419,6 +417,7 @@ class NRKTVEpisodeIE(InfoExtractor): 'params': { 'skip_download': True, }, + 'skip': 'ProgramRightsHasExpired', }] def _real_extract(self, url): From 1a005e8f1a59b6c00e1936292c575fb0f7bf711e Mon Sep 17 00:00:00 2001 From: Sergey M Date: Mon, 6 Jan 2020 06:29:28 +0700 Subject: [PATCH 9/9] Update nrk.py --- youtube_dl/extractor/nrk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py index 1d6f42e86..94115534b 100644 --- a/youtube_dl/extractor/nrk.py +++ b/youtube_dl/extractor/nrk.py @@ -265,7 +265,7 @@ class NRKTVIE(NRKBaseIE): 'title': 'Alarm Trolltunga', 'description': 'md5:46923a6e6510eefcce23d5ef2a58f2ce', 'duration': 2223, - 'age_limit': 6 + 'age_limit': 6, }, }, { 'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014', @@ -396,7 +396,7 @@ class NRKTVEpisodeIE(InfoExtractor): 'season_number': 1, 'episode_number': 2, 'episode': '2:6', - 'age_limit': 6 + 'age_limit': 6, }, 'params': { 'skip_download': True,