From a6246c1f5b67e9bab3d6408df6ef0a91528e4680 Mon Sep 17 00:00:00 2001 From: Petar Kukolj Date: Tue, 2 Oct 2018 18:28:46 +0200 Subject: [PATCH 1/6] [videofy.me] Fixing extractor to work after site redesign --- youtube_dl/extractor/videofyme.py | 72 ++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py index cd3f50a63..288ac16b3 100644 --- a/youtube_dl/extractor/videofyme.py +++ b/youtube_dl/extractor/videofyme.py @@ -1,9 +1,16 @@ from __future__ import unicode_literals +import json + from .common import InfoExtractor from ..utils import ( int_or_none, parse_iso8601, + unescapeHTML, + sanitize_url, + clean_html, + get_element_by_attribute, + js_to_json, ) @@ -11,42 +18,55 @@ class VideofyMeIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.videofy\.me/.+?|p\.videofy\.me/v)/(?P\d+)(&|#|$)' IE_NAME = 'videofy.me' - _TEST = { - 'url': 'http://www.videofy.me/thisisvideofyme/1100701', - 'md5': 'c77d700bdc16ae2e9f3c26019bd96143', + _TESTS = [{ + 'url': 'https://www.videofy.me/v/24582', + 'md5': '1e46140bacdae8959827903cecd054d9', 'info_dict': { - 'id': '1100701', + 'id': '24582', 'ext': 'mp4', - 'title': 'This is VideofyMe', - 'description': '', - 'upload_date': '20130326', - 'timestamp': 1364288959, - 'uploader': 'VideofyMe', - 'uploader_id': 'thisisvideofyme', + 'title': 'The VideofyMe app demo!', + 'description': 'This is VideofyMe.', + 'upload_date': '20120607', + 'timestamp': 1339070671, + 'uploader': 'oskarglauser', + 'uploader_id': 7010, 'view_count': int, - 'likes': int, - 'comment_count': int, }, - } + }, { + 'url': 'https://www.videofy.me/v/2975905', + 'md5': '79ad4498ab14dec72e815a8f85c7641c', + 'info_dict': { + 'id': '2975905', + 'ext': 'mp4', + 'title': 'But', + 'description': '', + 'upload_date': '20180126', + 'timestamp': 1516931131, + 'uploader': 'iamatlien', + 'uploader_id': 1798214, + 'view_count': int, + }, + },] def _real_extract(self, url): video_id = self._match_id(url) - config = self._download_json('http://vf-player-info-loader.herokuapp.com/%s.json' % video_id, video_id)['videoinfo'] + page = self._download_webpage(url, video_id) - video = config.get('video') - blog = config.get('blog', {}) + video_info = json.loads(get_element_by_attribute('type', 'application/ld+json', page)) + + meta = self._download_json('https://www.videofy.me/wp-json/wp/v2/posts/%s' % video_id, video_id) + uploader_id = meta.get('author') + uploader_name = self._download_json('https://www.videofy.me/wp-json/wp/v2/users/%s' % uploader_id, uploader_id, fatal=False).get('name') return { 'id': video_id, - 'title': video['title'], - 'url': video['sources']['source']['url'], - 'thumbnail': video.get('thumb'), - 'description': video.get('description'), - 'timestamp': parse_iso8601(video.get('date')), - 'uploader': blog.get('name'), - 'uploader_id': blog.get('identifier'), - 'view_count': int_or_none(self._search_regex(r'([0-9]+)', video.get('views'), 'view count', fatal=False)), - 'likes': int_or_none(video.get('likes')), - 'comment_count': int_or_none(video.get('nrOfComments')), + 'title': video_info['name'], + 'url': video_info['contentUrl'], + 'thumbnail': video_info.get('thumbnailUrl'), + 'description': clean_html(video_info.get('description')), + 'timestamp': parse_iso8601(video_info.get('uploadDate')), + 'uploader_id': uploader_id, + 'uploader': uploader_name, + 'view_count': int_or_none(video_info.get('interactionCount')), } From c443152757b941a400c7c695f9015718a4b4b0d0 Mon Sep 17 00:00:00 2001 From: Petar Kukolj Date: Tue, 2 Oct 2018 18:36:33 +0200 Subject: [PATCH 2/6] [videofy.me] Fixed flake8 errors --- youtube_dl/extractor/videofyme.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py index 288ac16b3..2fde51e6a 100644 --- a/youtube_dl/extractor/videofyme.py +++ b/youtube_dl/extractor/videofyme.py @@ -6,11 +6,8 @@ from .common import InfoExtractor from ..utils import ( int_or_none, parse_iso8601, - unescapeHTML, - sanitize_url, clean_html, get_element_by_attribute, - js_to_json, ) @@ -33,7 +30,7 @@ class VideofyMeIE(InfoExtractor): 'view_count': int, }, }, { - 'url': 'https://www.videofy.me/v/2975905', + 'url': 'https://www.videofy.me/v/2975905', 'md5': '79ad4498ab14dec72e815a8f85c7641c', 'info_dict': { 'id': '2975905', @@ -46,7 +43,7 @@ class VideofyMeIE(InfoExtractor): 'uploader_id': 1798214, 'view_count': int, }, - },] + }] def _real_extract(self, url): video_id = self._match_id(url) @@ -57,7 +54,7 @@ class VideofyMeIE(InfoExtractor): meta = self._download_json('https://www.videofy.me/wp-json/wp/v2/posts/%s' % video_id, video_id) uploader_id = meta.get('author') - uploader_name = self._download_json('https://www.videofy.me/wp-json/wp/v2/users/%s' % uploader_id, uploader_id, fatal=False).get('name') + uploader_name = self._download_json('https://www.videofy.me/wp-json/wp/v2/users/%s' % uploader_id, uploader_id, fatal=False).get('name') return { 'id': video_id, From 89db5af0fae4499358c77ac87563ce2914d7ff58 Mon Sep 17 00:00:00 2001 From: Petar Kukolj Date: Thu, 4 Oct 2018 21:17:46 +0200 Subject: [PATCH 3/6] [videofy.me] Made requested changes --- youtube_dl/extractor/videofyme.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py index 2fde51e6a..d511d0e09 100644 --- a/youtube_dl/extractor/videofyme.py +++ b/youtube_dl/extractor/videofyme.py @@ -1,13 +1,9 @@ from __future__ import unicode_literals -import json - from .common import InfoExtractor from ..utils import ( int_or_none, - parse_iso8601, clean_html, - get_element_by_attribute, ) @@ -50,20 +46,19 @@ class VideofyMeIE(InfoExtractor): page = self._download_webpage(url, video_id) - video_info = json.loads(get_element_by_attribute('type', 'application/ld+json', page)) + video_info = self._search_json_ld(page, video_id) - meta = self._download_json('https://www.videofy.me/wp-json/wp/v2/posts/%s' % video_id, video_id) - uploader_id = meta.get('author') + uploader_id = self._download_json('https://www.videofy.me/wp-json/wp/v2/posts/%s' % video_id, video_id, fatal=False).get('author') uploader_name = self._download_json('https://www.videofy.me/wp-json/wp/v2/users/%s' % uploader_id, uploader_id, fatal=False).get('name') return { 'id': video_id, - 'title': video_info['name'], - 'url': video_info['contentUrl'], - 'thumbnail': video_info.get('thumbnailUrl'), + 'title': video_info['title'], + 'url': video_info['url'], + 'thumbnail': video_info.get('thumbnail'), 'description': clean_html(video_info.get('description')), - 'timestamp': parse_iso8601(video_info.get('uploadDate')), + 'timestamp': video_info.get('timestamp'), 'uploader_id': uploader_id, 'uploader': uploader_name, - 'view_count': int_or_none(video_info.get('interactionCount')), + 'view_count': int_or_none(video_info.get('view_count')), } From 33e90109e6714dab28ef4fe25c02e8dbc0a22af0 Mon Sep 17 00:00:00 2001 From: Petar Kukolj Date: Sat, 6 Oct 2018 19:00:03 +0200 Subject: [PATCH 4/6] [videofy.me] Fixed extractor breaking on failed uploader info extraction --- youtube_dl/extractor/videofyme.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py index d511d0e09..be1fea6c6 100644 --- a/youtube_dl/extractor/videofyme.py +++ b/youtube_dl/extractor/videofyme.py @@ -48,17 +48,13 @@ class VideofyMeIE(InfoExtractor): video_info = self._search_json_ld(page, video_id) - uploader_id = self._download_json('https://www.videofy.me/wp-json/wp/v2/posts/%s' % video_id, video_id, fatal=False).get('author') - uploader_name = self._download_json('https://www.videofy.me/wp-json/wp/v2/users/%s' % uploader_id, uploader_id, fatal=False).get('name') + uploader_id = self._download_json('https://www.videofy.me/wp-json/wp/v2/posts/%s' % video_id, video_id, fatal=False) + uploader_id = uploader_id.get('author') if uploader_id != False else None + uploader_name = self._download_json('https://www.videofy.me/wp-json/wp/v2/users/%s' % uploader_id, uploader_id, fatal=False) + uploader_name = uploader_name.get('name') if uploader_name != False else None - return { - 'id': video_id, - 'title': video_info['title'], - 'url': video_info['url'], - 'thumbnail': video_info.get('thumbnail'), - 'description': clean_html(video_info.get('description')), - 'timestamp': video_info.get('timestamp'), - 'uploader_id': uploader_id, - 'uploader': uploader_name, - 'view_count': int_or_none(video_info.get('view_count')), - } + video_info['id'] = video_id + video_info['uploader_id'] = uploader_id + video_info['uploader'] = uploader_name + + return video_info From 532163567931b63a7300221ac5b6eac94bd77755 Mon Sep 17 00:00:00 2001 From: Petar Kukolj Date: Sat, 6 Oct 2018 19:05:13 +0200 Subject: [PATCH 5/6] [videofy.me] Fixed flake8 errors --- youtube_dl/extractor/videofyme.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py index be1fea6c6..2998ec03f 100644 --- a/youtube_dl/extractor/videofyme.py +++ b/youtube_dl/extractor/videofyme.py @@ -1,10 +1,6 @@ from __future__ import unicode_literals from .common import InfoExtractor -from ..utils import ( - int_or_none, - clean_html, -) class VideofyMeIE(InfoExtractor): @@ -48,13 +44,15 @@ class VideofyMeIE(InfoExtractor): video_info = self._search_json_ld(page, video_id) - uploader_id = self._download_json('https://www.videofy.me/wp-json/wp/v2/posts/%s' % video_id, video_id, fatal=False) - uploader_id = uploader_id.get('author') if uploader_id != False else None + uploader_id = self._download_json('https://www.videofy.me/wp-json/wp/v2/poss/%s' % video_id, video_id, fatal=False) + uploader_id = uploader_id.get('author') if uploader_id is not False else None uploader_name = self._download_json('https://www.videofy.me/wp-json/wp/v2/users/%s' % uploader_id, uploader_id, fatal=False) - uploader_name = uploader_name.get('name') if uploader_name != False else None + uploader_name = uploader_name.get('name') if uploader_name is not False else None video_info['id'] = video_id video_info['uploader_id'] = uploader_id video_info['uploader'] = uploader_name + print(video_info) + return video_info From 332aac3975668ba60ed1e6239c33e1e29ac55147 Mon Sep 17 00:00:00 2001 From: Petar Kukolj Date: Sat, 6 Oct 2018 19:08:14 +0200 Subject: [PATCH 6/6] [videofy.me] Fixed a flake8 error --- youtube_dl/extractor/videofyme.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py index 2998ec03f..d95fcb56b 100644 --- a/youtube_dl/extractor/videofyme.py +++ b/youtube_dl/extractor/videofyme.py @@ -53,6 +53,4 @@ class VideofyMeIE(InfoExtractor): video_info['uploader_id'] = uploader_id video_info['uploader'] = uploader_name - print(video_info) - return video_info