From 665d7844946f86c8a695ad21932506c9b03ca34d Mon Sep 17 00:00:00 2001 From: Yonghui Chen Date: Mon, 15 Jan 2018 17:17:21 +0800 Subject: [PATCH 1/3] [ifeng] Add new extractor --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/ifeng.py | 71 ++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 youtube_dl/extractor/ifeng.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index c82614bf9..ad4d3980f 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -445,6 +445,7 @@ from .huajiao import HuajiaoIE from .huffpost import HuffPostIE from .hypem import HypemIE from .iconosquare import IconosquareIE +from .ifeng import IFengIE from .ign import ( IGNIE, OneUPIE, diff --git a/youtube_dl/extractor/ifeng.py b/youtube_dl/extractor/ifeng.py new file mode 100644 index 000000000..abada4dee --- /dev/null +++ b/youtube_dl/extractor/ifeng.py @@ -0,0 +1,71 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + +from ..utils import ( + ExtractorError, + unified_strdate, +) + + +class IFengIE(InfoExtractor): + _VALID_URL = r'https?://(?:v\.)?ifeng\.com/video_(?P\d+)\.shtml' + + _TEST = { + 'url': 'http://v.ifeng.com/video_11129877.shtml', + 'md5': '6cdca783b34bd31a678f418f4ff60816', + 'info_dict': { + 'id': '11129877', + 'ext': 'mp4', + 'title': '能自我修复的玻璃，手机用上它再也不怕碎屏了！', + 'description': 'md5:03a4d6c8c5ac7140b0cb085309fa8992', + 'timestamp': 1515675654, + 'upload_date': '20180111', + } + } + + def _report_error(self, result): + if 'message' in result: + raise ExtractorError('%s said: %s' % (self.IE_NAME, result['message']), expected=True) + elif 'code' in result: + raise ExtractorError('%s returns error %d' % (self.IE_NAME, result['code']), expected=True) + else: + raise ExtractorError('Can\'t extract Bangumi episode ID') + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + title = self._html_search_regex( + r'"name": "(?P(.+?))"', + webpage, 'title', group='value') + video_url = self._html_search_regex( + r'"videoPlayUrl": "(?P(.+?))"', + webpage, 'url', group='value') + + if not video_url: + self._report_error(title) + + thumbnail = self._html_search_regex( + r'"videoLargePoster": "(?P(.+?))"', + webpage, 'thumbnail', group='value', fatal=False) + uploader = self._html_search_regex( + r'"columnName":"(?P(.+?))"', + webpage, 'uploader', group='value', fatal=False) + duration = self._html_search_regex( + r'"duration": "(?P(.+?))"', + webpage, 'duration', group='value', fatal=False) + upload_date = unified_strdate(self._html_search_regex( + r'"createdate": "(?P(.+?))"', + webpage, 'createdate', group='value', fatal=False)) + + return { + 'id': video_id, + 'title': title, + 'duration': int(duration), + 'uploader': uploader, + 'upload_date': upload_date, + 'url': video_url, + 'thumbnail': thumbnail + } From 76c5ae8e801334b9d74e45517f2f3da9ba8dfbca Mon Sep 17 00:00:00 2001 From: Yonghui Chen Date: Tue, 16 Jan 2018 17:10:10 +0800 Subject: [PATCH 2/3] [ifeng] Change to extract videoinfo --- youtube_dl/extractor/ifeng.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/ifeng.py b/youtube_dl/extractor/ifeng.py index abada4dee..56a4ca16d 100644 --- a/youtube_dl/extractor/ifeng.py +++ b/youtube_dl/extractor/ifeng.py @@ -60,12 +60,20 @@ class IFengIE(InfoExtractor): r'"createdate": "(?P(.+?))"', webpage, 'createdate', group='value', fatal=False)) + formats = [ + { + 'url': video_url, + 'ext': video_url[video_url.rfind('.') + 1:], + } + ] + self._sort_formats(formats) + return { 'id': video_id, 'title': title, 'duration': int(duration), 'uploader': uploader, 'upload_date': upload_date, - 'url': video_url, - 'thumbnail': thumbnail + 'thumbnail': thumbnail, + 'formats': formats, } From b95dff51255eceb60463abf3f1b95156f630febd Mon Sep 17 00:00:00 2001 From: Yonghui Chen Date: Thu, 18 Jan 2018 15:05:49 +0800 Subject: [PATCH 3/3] [ifeng] JavaScript object transformed to a Python dictionary instead of a long regular expression --- youtube_dl/extractor/ifeng.py | 38 ++++++++++++----------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/youtube_dl/extractor/ifeng.py b/youtube_dl/extractor/ifeng.py index 56a4ca16d..8c8b19985 100644 --- a/youtube_dl/extractor/ifeng.py +++ b/youtube_dl/extractor/ifeng.py @@ -6,7 +6,9 @@ from .common import InfoExtractor from ..utils import ( ExtractorError, unified_strdate, + int_or_none ) +import re class IFengIE(InfoExtractor): @@ -37,28 +39,14 @@ class IFengIE(InfoExtractor): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - title = self._html_search_regex( - r'"name": "(?P(.+?))"', - webpage, 'title', group='value') - video_url = self._html_search_regex( - r'"videoPlayUrl": "(?P(.+?))"', - webpage, 'url', group='value') + video_info = self._parse_json(self._html_search_regex( + r'var videoinfo = (?P({.+?}));', + webpage, 'video_info', flags=re.DOTALL, group='value'), video_id) + + video_url = video_info.get('videoPlayUrl') if not video_url: - self._report_error(title) - - thumbnail = self._html_search_regex( - r'"videoLargePoster": "(?P(.+?))"', - webpage, 'thumbnail', group='value', fatal=False) - uploader = self._html_search_regex( - r'"columnName":"(?P(.+?))"', - webpage, 'uploader', group='value', fatal=False) - duration = self._html_search_regex( - r'"duration": "(?P(.+?))"', - webpage, 'duration', group='value', fatal=False) - upload_date = unified_strdate(self._html_search_regex( - r'"createdate": "(?P(.+?))"', - webpage, 'createdate', group='value', fatal=False)) + self._report_error(video_url) formats = [ { @@ -70,10 +58,10 @@ class IFengIE(InfoExtractor): return { 'id': video_id, - 'title': title, - 'duration': int(duration), - 'uploader': uploader, - 'upload_date': upload_date, - 'thumbnail': thumbnail, + 'title': video_info.get('name'), + 'duration': int_or_none(video_info.get('duration')), + 'uploader': video_info.get('columnName'), + 'upload_date': unified_strdate(video_info.get('createdate')), + 'thumbnail': video_info.get('videoLargePoster'), 'formats': formats, }