From 11b9ebc2edcec9bf8de047cb969d7a3218ad51d0 Mon Sep 17 00:00:00 2001 From: Zack Fernandes Date: Sun, 31 Dec 2017 13:55:35 -0800 Subject: [PATCH 1/2] [Tumblr] Add login support --- youtube_dl/extractor/tumblr.py | 35 +++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py index 786143525..68682b15d 100644 --- a/youtube_dl/extractor/tumblr.py +++ b/youtube_dl/extractor/tumblr.py @@ -4,11 +4,19 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..utils import int_or_none +from ..utils import ( + ExtractorError, + int_or_none, + sanitized_Request, + urlencode_postdata +) class TumblrIE(InfoExtractor): _VALID_URL = r'https?://(?P[^/?#&]+)\.tumblr\.com/(?:post|video)/(?P[0-9]+)(?:$|[/?#])' + _NETRC_MACHINE = 'tumblr' + _LOGIN_REQUIRED = False + _LOGIN_URL = 'https://www.tumblr.com/login' _TESTS = [{ 'url': 'http://tatianamaslanydaily.tumblr.com/post/54196191430/orphan-black-dvd-extra-behind-the-scenes', 'md5': '479bb068e5b16462f5176a6828829767', @@ -97,6 +105,31 @@ class TumblrIE(InfoExtractor): 'add_ie': ['Instagram'], }] + def _real_initialize(self): + self._login() + + def _login(self): + (username, password) = self._get_login_info() + if username is None: + return + self.report_login() + webpage = self._download_webpage(self._LOGIN_URL, None, False) + form = self._hidden_inputs(webpage) + form.update({ + 'user[email]': username, + 'user[password]': password + }) + post_data = urlencode_postdata(form) + login_request = sanitized_Request(self._LOGIN_URL, post_data) + login_request.add_header('Content-Type', 'application/x-www-form-urlencoded') + login_request.add_header('Referer', self._LOGIN_URL) + login_response = self._download_webpage(login_request, None, False, 'Wrong login info') + + # Check the login response from Tumblr for an error message and fail the extraction if we find one. + login_errors = self._search_regex(r'Tumblr\.RegistrationForm\.errors = \[(.*)\]', login_response, 'login errors', False, False) + if login_errors: + raise ExtractorError("Error logging in: %s" % login_errors) + def _real_extract(self, url): m_url = re.match(self._VALID_URL, url) video_id = m_url.group('id') From b038ec285a47e0715a19bf6a20d700d9e89faf86 Mon Sep 17 00:00:00 2001 From: Zack Fernandes Date: Sat, 6 Jan 2018 14:33:05 -0800 Subject: [PATCH 2/2] [Tumblr] Cleanup login support code after code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Removed unused _LOGIN_REQUIRED constant - Moved data and headers into login_response _download_webpage call - Made login_errors RegEx a little more versatile: relaxed whitespace, support single and double quotes in the array, don’t match an empty string. - Use single quotes and add expected to ExtractorError --- youtube_dl/extractor/tumblr.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py index 68682b15d..58ac66755 100644 --- a/youtube_dl/extractor/tumblr.py +++ b/youtube_dl/extractor/tumblr.py @@ -15,7 +15,6 @@ from ..utils import ( class TumblrIE(InfoExtractor): _VALID_URL = r'https?://(?P[^/?#&]+)\.tumblr\.com/(?:post|video)/(?P[0-9]+)(?:$|[/?#])' _NETRC_MACHINE = 'tumblr' - _LOGIN_REQUIRED = False _LOGIN_URL = 'https://www.tumblr.com/login' _TESTS = [{ 'url': 'http://tatianamaslanydaily.tumblr.com/post/54196191430/orphan-black-dvd-extra-behind-the-scenes', @@ -119,16 +118,16 @@ class TumblrIE(InfoExtractor): 'user[email]': username, 'user[password]': password }) - post_data = urlencode_postdata(form) - login_request = sanitized_Request(self._LOGIN_URL, post_data) - login_request.add_header('Content-Type', 'application/x-www-form-urlencoded') - login_request.add_header('Referer', self._LOGIN_URL) - login_response = self._download_webpage(login_request, None, False, 'Wrong login info') + login_response = self._download_webpage( + sanitized_Request(self._LOGIN_URL, urlencode_postdata(form), { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Referer': self._LOGIN_URL + }), None, False, 'Wrong login info') # Check the login response from Tumblr for an error message and fail the extraction if we find one. - login_errors = self._search_regex(r'Tumblr\.RegistrationForm\.errors = \[(.*)\]', login_response, 'login errors', False, False) + login_errors = self._search_regex(r'Tumblr\.RegistrationForm\.errors\s*=\s*\[[\"|\'](.+)[\"|\']\]', login_response, 'login errors', False) if login_errors: - raise ExtractorError("Error logging in: %s" % login_errors) + raise ExtractorError('Error logging in: %s' % login_errors, expected=True) def _real_extract(self, url): m_url = re.match(self._VALID_URL, url)