From 7cb0a7418a0e2b162b9e0af63cfd95264f531d97 Mon Sep 17 00:00:00 2001 From: Tyler Romeo Date: Sat, 6 Aug 2016 16:00:37 -0400 Subject: [PATCH] [crunchyroll] Grab mobile API formats Merge in livestreamer's functionality of fetching Crunchyroll video information from their mobile API, which contains an independent set of streams from the desktop API. Grab the additional formats, process them, and sort them accordingly. --- youtube_dl/extractor/crunchyroll.py | 153 +++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index 6d3abb52f..013399fd6 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -1,7 +1,9 @@ # encoding: utf-8 from __future__ import unicode_literals +import random import re +import string import json import base64 import zlib @@ -36,10 +38,18 @@ from ..aes import ( class CrunchyrollBaseIE(InfoExtractor): _NETRC_MACHINE = 'crunchyroll' + def __init__(self, *args, **kwargs): + super(CrunchyrollBaseIE, self).__init__(*args, **kwargs) + self.api_session_id = None + self.api_session_auth = None + self.api_device_id = ''.join(random.sample(string.ascii_letters + string.digits, 32)) + def _login(self): (username, password) = self._get_login_info() if username is None: return + + # Log into main website self.report_login() login_url = 'https://www.crunchyroll.com/?a=formhandler' data = urlencode_postdata({ @@ -51,6 +61,41 @@ class CrunchyrollBaseIE(InfoExtractor): login_request.add_header('Content-Type', 'application/x-www-form-urlencoded') self._download_webpage(login_request, None, False, 'Wrong login info') + # Start session with mobile API + res = self._api_call( + 'start_session', + { + 'device_id': self.api_device_id, + 'device_type': 'com.crunchyroll.iphone', + 'access_token': 'QWjz212GspMHH9h' + }, + video_id=None, + note='Starting session with mobile API', + errnote='Could not start session with mobile API', + # If mobile API fails, we can always fall back on the regular website + fatal=False + ) + + if res is not None: + self.api_session_id = res.get('data', {'session_id': None}).get('session_id') + + # Log into mobile API + res = self._api_call( + 'login', + { + 'account': username, + 'password': password + }, + video_id=None, + note='Logging in to mobile API', + errnote='Could not log into mobile API', + # If mobile API fails, we can always fall back on the regular website + fatal=False + ) + + if res is not None: + self.api_session_auth = res.get('data', {'auth': None}).get('auth') + def _real_initialize(self): self._login() @@ -67,6 +112,42 @@ class CrunchyrollBaseIE(InfoExtractor): request.add_header('Accept-Language', '*') return super(CrunchyrollBaseIE, self)._download_webpage(request, *args, **kwargs) + def _api_call(self, entrypoint, params, video_id, *args, **kwargs): + '''Makes a call against the api. + + :param entrypoint: API method to call. + :param params: parameters to include in the request data. + :param schema: schema to use to validate the data + + Source adapted from: + https://github.com/chrippa/livestreamer/blob/develop/src/livestreamer/plugins/crunchyroll.py + Copyright (c) 2011-2015, Christopher Rosell + License: https://github.com/chrippa/livestreamer/blob/develop/LICENSE + ''' + + url = 'https://api.crunchyroll.com/{0}.0.json'.format(entrypoint) + + # Default params + params = dict(params) + params.update({ + 'version': '2313.8', + 'locale': 'enUS', + }) + + if self.api_session_id: + params["session_id"] = self.api_session_id + + # Headers + headers = { + 'Host': 'api.crunchyroll.com', + 'Accept-Encoding': 'gzip, deflzate', + 'Accept': '*/*', + 'Content-Type': 'application/x-www-form-urlencoded' + } + headers['User-Agent'] = 'Mozilla/5.0 (iPhone; iPhone OS 8.3.0; en_US)' + + return self._download_json(url, video_id, query=params, headers=headers, *args, **kwargs) + @staticmethod def _add_skip_wall(url): parsed_url = compat_urlparse.urlparse(url) @@ -87,7 +168,7 @@ class CrunchyrollIE(CrunchyrollBaseIE): 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513', 'info_dict': { 'id': '645513', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!', 'description': 'md5:2d17137920c64f2f49981a7797d275ef', 'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg', @@ -96,7 +177,7 @@ class CrunchyrollIE(CrunchyrollBaseIE): 'url': 're:(?!.*&)', }, 'params': { - # rtmp + # m3u8 download 'skip_download': True, }, }, { @@ -145,6 +226,22 @@ class CrunchyrollIE(CrunchyrollBaseIE): '1080': ('80', '108'), } + # Crunchyroll does not give us bitrate data for the RTMP sources, + # so by default self._sort_formats() will put the HLS streams first. + # However, the HLS streams are actually a lower bitrate than their + # RTMP counterparts. + _FORMAT_QUALITY = { + 'ultralow': 2, + 'low': 3, + '360p': 4, + 'mid': 5, + '480p': 6, + 'high': 7, + '720p': 8, + 'ultra': 9, + '1080p': 10 + } + def _decrypt_subtitles(self, data, iv, id): data = bytes_to_intlist(base64.b64decode(data.encode('utf-8'))) iv = bytes_to_intlist(base64.b64decode(iv.encode('utf-8'))) @@ -281,6 +378,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text mobj = re.match(self._VALID_URL, url) video_id = mobj.group('video_id') + # Fetch mobile webpage if mobj.group('prefix') == 'm': mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage') webpage_url = self._search_regex(r'', mobile_webpage, 'webpage_url') @@ -303,6 +401,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text if 'To view this, please log in to verify you are 18 or older.' in webpage: self.raise_login_required() + # Extract title, description, and other metadata + # from the mobile pagedlist video_title = self._html_search_regex( r'(?s)]*>((?:(?!]+itemprop=["\']title["\'][^>]*>(?:(?!', webpage, 'video_title') @@ -321,6 +421,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text r']+href="/publisher/[^"]+"[^>]*>([^<]+)', webpage, 'video_uploader', fatal=False) + # Extract the available RTMP formats available_fmts = [] for a, fmt in re.findall(r'(]+token=["\']showmedia\.([0-9]{3,4})p["\'][^>]+>)', webpage): attrs = extract_attributes(a) @@ -335,6 +436,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text break video_encode_ids = [] formats = [] + # Process the RTMP formats for fmt in available_fmts: stream_quality, stream_format = self._FORMAT_IDS[fmt] video_format = fmt + 'p' @@ -347,6 +449,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text streamdata_req, video_id, note='Downloading media info for %s' % video_format) stream_info = streamdata.find('./{default}preload/stream_info') + video_encode_id = xpath_text(stream_info, './video_encode_id') if video_encode_id in video_encode_ids: continue @@ -367,9 +470,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text metadata = stream_info.find('./metadata') format_info = { 'format': video_format, - 'format_id': video_format, + 'format_id': 'rtmp-' + video_format, 'height': int_or_none(xpath_text(metadata, './height')), 'width': int_or_none(xpath_text(metadata, './width')), + 'quality': self._FORMAT_QUALITY[video_format] } if '.fplive.net/' in video_url: @@ -391,6 +495,49 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text 'ext': 'flv', }) formats.append(format_info) + + # Fetch HLS formats from the API + api_info = self._api_call('info', { + 'media_id': video_id, + 'fields': 'media.stream_data' + }, video_id) + + available_fmts = (api_info + .get('data', {'stream_data': {'streams': []}}) + .get('stream_data', {'streams': []}) + .get('streams', [])) + for fmt in available_fmts: + # Crunchyroll has a mysterious format with quality None, that is + # lower resolution than even "low" quality + if fmt['quality'] is None: + fmt['quality'] = 'ultralow' + # Ignore the "adaptive" format, which is just a single HLS URL + # that duplicates all the others + elif fmt['quality'] == 'adaptive': + continue + + m3u8_formats = self._extract_m3u8_formats( + fmt.get('url'), + video_id, + ) + + for m3u8_fmt in m3u8_formats: + m3u8_fmt['quality'] = self._FORMAT_QUALITY[fmt['quality']] + + if m3u8_fmt['format_id']: + m3u8_fmt['format_id'] = 'hls-%s-%s' % (fmt['quality'], m3u8_fmt['format_id']) + else: + m3u8_fmt['format_id'] = 'hls-%s' % (fmt['quality']) + + if fmt.get('height') is not None: + m3u8_fmt['height'] = int_or_none(fmt['height']) + if fmt.get('width') is not None: + m3u8_fmt['width'] = int_or_none(fmt['width']) + if fmt.get('bitrate') is not None: + m3u8_fmt['tbr'] = int_or_none(fmt['bitrate']) + + formats.extend(m3u8_formats) + self._sort_formats(formats) metadata = self._download_xml(