mirror of
https://github.com/l1ving/youtube-dl
synced 2025-03-07 14:37:24 +08:00
Merge remote branch 'rg3/master' into ytuser-infoextractor
Conflicts: youtube-dl
This commit is contained in:
Paweł Paprota
commit
6513cc1739
277
youtube-dl
277
youtube-dl
@ -4,9 +4,12 @@
|
||||
# Author: Danny Colligan
|
||||
# Author: Benjamin Johnson
|
||||
# Author: Vasyl' Vavrychuk
|
||||
# Author: Witold Baryluk
|
||||
# License: Public domain code
|
||||
import cookielib
|
||||
import ctypes
|
||||
import datetime
|
||||
import gzip
|
||||
import htmlentitydefs
|
||||
import httplib
|
||||
import locale
|
||||
@ -17,11 +20,13 @@ import os.path
|
||||
import re
|
||||
import socket
|
||||
import string
|
||||
import StringIO
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib
|
||||
import urllib2
|
||||
import zlib
|
||||
|
||||
# parse_qs was moved from the cgi module to the urlparse module recently.
|
||||
try:
|
||||
@ -57,7 +62,7 @@ def preferredencoding():
|
||||
|
||||
def htmlentity_transform(matchobj):
|
||||
"""Transforms an HTML entity to a Unicode character.
|
||||
|
||||
|
||||
This function receives a match object and is intended to be used with
|
||||
the re.sub() function.
|
||||
"""
|
||||
@ -114,7 +119,7 @@ def sanitize_open(filename, open_mode):
|
||||
|
||||
class DownloadError(Exception):
|
||||
"""Download Error exception.
|
||||
|
||||
|
||||
This exception may be thrown by FileDownloader objects if they are not
|
||||
configured to continue on errors. They will contain the appropriate
|
||||
error message.
|
||||
@ -160,6 +165,64 @@ class ContentTooShortError(Exception):
|
||||
self.downloaded = downloaded
|
||||
self.expected = expected
|
||||
|
||||
class YoutubeDLHandler(urllib2.HTTPHandler):
|
||||
"""Handler for HTTP requests and responses.
|
||||
|
||||
This class, when installed with an OpenerDirector, automatically adds
|
||||
the standard headers to every HTTP request and handles gzipped and
|
||||
deflated responses from web servers. If compression is to be avoided in
|
||||
a particular request, the original request in the program code only has
|
||||
to include the HTTP header "Youtubedl-No-Compression", which will be
|
||||
removed before making the real request.
|
||||
|
||||
Part of this code was copied from:
|
||||
|
||||
http://techknack.net/python-urllib2-handlers/
|
||||
|
||||
Andrew Rowls, the author of that code, agreed to release it to the
|
||||
public domain.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def deflate(data):
|
||||
try:
|
||||
return zlib.decompress(data, -zlib.MAX_WBITS)
|
||||
except zlib.error:
|
||||
return zlib.decompress(data)
|
||||
|
||||
@staticmethod
|
||||
def addinfourl_wrapper(stream, headers, url, code):
|
||||
if hasattr(urllib2.addinfourl, 'getcode'):
|
||||
return urllib2.addinfourl(stream, headers, url, code)
|
||||
ret = urllib2.addinfourl(stream, headers, url)
|
||||
ret.code = code
|
||||
return ret
|
||||
|
||||
def http_request(self, req):
|
||||
for h in std_headers:
|
||||
if h in req.headers:
|
||||
del req.headers[h]
|
||||
req.add_header(h, std_headers[h])
|
||||
if 'Youtubedl-no-compression' in req.headers:
|
||||
if 'Accept-encoding' in req.headers:
|
||||
del req.headers['Accept-encoding']
|
||||
del req.headers['Youtubedl-no-compression']
|
||||
return req
|
||||
|
||||
def http_response(self, req, resp):
|
||||
old_resp = resp
|
||||
# gzip
|
||||
if resp.headers.get('Content-encoding', '') == 'gzip':
|
||||
gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
|
||||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
||||
resp.msg = old_resp.msg
|
||||
# deflate
|
||||
if resp.headers.get('Content-encoding', '') == 'deflate':
|
||||
gz = StringIO.StringIO(self.deflate(resp.read()))
|
||||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
||||
resp.msg = old_resp.msg
|
||||
return resp
|
||||
|
||||
class FileDownloader(object):
|
||||
"""File Downloader class.
|
||||
|
||||
@ -208,6 +271,8 @@ class FileDownloader(object):
|
||||
playliststart: Playlist item to start at.
|
||||
playlistend: Playlist item to end at.
|
||||
logtostderr: Log messages to stderr instead of stdout.
|
||||
consoletitle: Display progress in console window's titlebar.
|
||||
nopart: Do not use temporary .part files.
|
||||
"""
|
||||
|
||||
params = None
|
||||
@ -225,7 +290,7 @@ class FileDownloader(object):
|
||||
self._num_downloads = 0
|
||||
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
|
||||
self.params = params
|
||||
|
||||
|
||||
@staticmethod
|
||||
def pmkdir(filename):
|
||||
"""Create directory components in filename. Similar to Unix "mkdir -p"."""
|
||||
@ -235,14 +300,7 @@ class FileDownloader(object):
|
||||
for dir in aggregate:
|
||||
if not os.path.exists(dir):
|
||||
os.mkdir(dir)
|
||||
|
||||
@staticmethod
|
||||
def temp_name(filename):
|
||||
"""Returns a temporary filename for the given filename."""
|
||||
if filename == u'-' or (os.path.exists(filename) and not os.path.isfile(filename)):
|
||||
return filename
|
||||
return filename + u'.part'
|
||||
|
||||
|
||||
@staticmethod
|
||||
def format_bytes(bytes):
|
||||
if bytes is None:
|
||||
@ -311,12 +369,12 @@ class FileDownloader(object):
|
||||
"""Add an InfoExtractor object to the end of the list."""
|
||||
self._ies.append(ie)
|
||||
ie.set_downloader(self)
|
||||
|
||||
|
||||
def add_post_processor(self, pp):
|
||||
"""Add a PostProcessor object to the end of the chain."""
|
||||
self._pps.append(pp)
|
||||
pp.set_downloader(self)
|
||||
|
||||
|
||||
def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False):
|
||||
"""Print message to stdout if not in quiet mode."""
|
||||
try:
|
||||
@ -327,11 +385,22 @@ class FileDownloader(object):
|
||||
except (UnicodeEncodeError), err:
|
||||
if not ignore_encoding_errors:
|
||||
raise
|
||||
|
||||
|
||||
def to_stderr(self, message):
|
||||
"""Print message to stderr."""
|
||||
print >>sys.stderr, message.encode(preferredencoding())
|
||||
|
||||
|
||||
def to_cons_title(self, message):
|
||||
"""Set console/terminal window title to message."""
|
||||
if not self.params.get('consoletitle', False):
|
||||
return
|
||||
if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
|
||||
# c_wchar_p() might not be necessary if `message` is
|
||||
# already of type unicode()
|
||||
ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
|
||||
elif 'TERM' in os.environ:
|
||||
sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
|
||||
|
||||
def fixed_template(self):
|
||||
"""Checks if the output template is fixed."""
|
||||
return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
|
||||
@ -361,7 +430,19 @@ class FileDownloader(object):
|
||||
speed = float(byte_counter) / elapsed
|
||||
if speed > rate_limit:
|
||||
time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
|
||||
|
||||
|
||||
def temp_name(self, filename):
|
||||
"""Returns a temporary filename for the given filename."""
|
||||
if self.params.get('nopart', False) or filename == u'-' or \
|
||||
(os.path.exists(filename) and not os.path.isfile(filename)):
|
||||
return filename
|
||||
return filename + u'.part'
|
||||
|
||||
def undo_temp_name(self, filename):
|
||||
if filename.endswith(u'.part'):
|
||||
return filename[:-len(u'.part')]
|
||||
return filename
|
||||
|
||||
def try_rename(self, old_filename, new_filename):
|
||||
try:
|
||||
if old_filename == new_filename:
|
||||
@ -373,40 +454,42 @@ class FileDownloader(object):
|
||||
def report_destination(self, filename):
|
||||
"""Report destination filename."""
|
||||
self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
|
||||
|
||||
|
||||
def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
|
||||
"""Report download progress."""
|
||||
if self.params.get('noprogress', False):
|
||||
return
|
||||
self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
|
||||
(percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
|
||||
self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
|
||||
(percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
|
||||
|
||||
def report_resuming_byte(self, resume_len):
|
||||
"""Report attempt to resume at given byte."""
|
||||
self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
|
||||
|
||||
|
||||
def report_retry(self, count, retries):
|
||||
"""Report retry in case of HTTP error 5xx"""
|
||||
self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
|
||||
|
||||
|
||||
def report_file_already_downloaded(self, file_name):
|
||||
"""Report file has already been fully downloaded."""
|
||||
try:
|
||||
self.to_screen(u'[download] %s has already been downloaded' % file_name)
|
||||
except (UnicodeEncodeError), err:
|
||||
self.to_screen(u'[download] The file has already been downloaded')
|
||||
|
||||
|
||||
def report_unable_to_resume(self):
|
||||
"""Report it was impossible to resume download."""
|
||||
self.to_screen(u'[download] Unable to resume')
|
||||
|
||||
|
||||
def report_finish(self):
|
||||
"""Report download finished."""
|
||||
if self.params.get('noprogress', False):
|
||||
self.to_screen(u'[download] Download completed')
|
||||
else:
|
||||
self.to_screen(u'')
|
||||
|
||||
|
||||
def increment_downloads(self):
|
||||
"""Increment the ordinal that assigns a number to each file."""
|
||||
self._num_downloads += 1
|
||||
@ -426,7 +509,7 @@ class FileDownloader(object):
|
||||
print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
|
||||
|
||||
return
|
||||
|
||||
|
||||
try:
|
||||
template_dict = dict(info_dict)
|
||||
template_dict['epoch'] = unicode(long(time.time()))
|
||||
@ -497,7 +580,7 @@ class FileDownloader(object):
|
||||
info = pp.run(info)
|
||||
if info is None:
|
||||
break
|
||||
|
||||
|
||||
def _download_with_rtmpdump(self, filename, url, player_url):
|
||||
self.report_destination(filename)
|
||||
tmpfilename = self.temp_name(filename)
|
||||
@ -532,7 +615,7 @@ class FileDownloader(object):
|
||||
|
||||
def _do_download(self, filename, url, player_url):
|
||||
# Check file already present
|
||||
if self.params.get('continuedl', False) and os.path.isfile(filename):
|
||||
if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
|
||||
self.report_file_already_downloaded(filename)
|
||||
return True
|
||||
|
||||
@ -543,8 +626,11 @@ class FileDownloader(object):
|
||||
tmpfilename = self.temp_name(filename)
|
||||
stream = None
|
||||
open_mode = 'wb'
|
||||
basic_request = urllib2.Request(url, None, std_headers)
|
||||
request = urllib2.Request(url, None, std_headers)
|
||||
|
||||
# Do not include the Accept-Encoding header
|
||||
headers = {'Youtubedl-no-compression': 'True'}
|
||||
basic_request = urllib2.Request(url, None, headers)
|
||||
request = urllib2.Request(url, None, headers)
|
||||
|
||||
# Establish possible resume length
|
||||
if os.path.isfile(tmpfilename):
|
||||
@ -626,6 +712,7 @@ class FileDownloader(object):
|
||||
if stream is None:
|
||||
try:
|
||||
(stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
|
||||
filename = self.undo_temp_name(tmpfilename)
|
||||
self.report_destination(filename)
|
||||
except (OSError, IOError), err:
|
||||
self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
|
||||
@ -715,7 +802,7 @@ class InfoExtractor(object):
|
||||
def set_downloader(self, downloader):
|
||||
"""Sets the downloader for this IE."""
|
||||
self._downloader = downloader
|
||||
|
||||
|
||||
def _real_initialize(self):
|
||||
"""Real initialization process. Redefine in subclasses."""
|
||||
pass
|
||||
@ -727,7 +814,7 @@ class InfoExtractor(object):
|
||||
class YoutubeIE(InfoExtractor):
|
||||
"""Information extractor for youtube.com."""
|
||||
|
||||
_VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/(?:(?:v/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))))?([0-9A-Za-z_-]+)(?(1).+)?$'
|
||||
_VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
|
||||
_LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
|
||||
_LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
|
||||
_AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
|
||||
@ -756,31 +843,31 @@ class YoutubeIE(InfoExtractor):
|
||||
def report_login(self):
|
||||
"""Report attempt to log in."""
|
||||
self._downloader.to_screen(u'[youtube] Logging in')
|
||||
|
||||
|
||||
def report_age_confirmation(self):
|
||||
"""Report attempt to confirm age."""
|
||||
self._downloader.to_screen(u'[youtube] Confirming age')
|
||||
|
||||
|
||||
def report_video_webpage_download(self, video_id):
|
||||
"""Report attempt to download video webpage."""
|
||||
self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
|
||||
|
||||
|
||||
def report_video_info_webpage_download(self, video_id):
|
||||
"""Report attempt to download video info webpage."""
|
||||
self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
|
||||
|
||||
|
||||
def report_information_extraction(self, video_id):
|
||||
"""Report attempt to extract video information."""
|
||||
self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
|
||||
|
||||
|
||||
def report_unavailable_format(self, video_id, format):
|
||||
"""Report extracted video URL."""
|
||||
self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
|
||||
|
||||
|
||||
def report_rtmp_download(self):
|
||||
"""Indicate the download will use the RTMP protocol."""
|
||||
self._downloader.to_screen(u'[youtube] RTMP download detected')
|
||||
|
||||
|
||||
def _real_initialize(self):
|
||||
if self._downloader is None:
|
||||
return
|
||||
@ -806,7 +893,7 @@ class YoutubeIE(InfoExtractor):
|
||||
return
|
||||
|
||||
# Set language
|
||||
request = urllib2.Request(self._LANG_URL, None, std_headers)
|
||||
request = urllib2.Request(self._LANG_URL)
|
||||
try:
|
||||
self.report_lang()
|
||||
urllib2.urlopen(request).read()
|
||||
@ -826,7 +913,7 @@ class YoutubeIE(InfoExtractor):
|
||||
'username': username,
|
||||
'password': password,
|
||||
}
|
||||
request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
|
||||
request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
|
||||
try:
|
||||
self.report_login()
|
||||
login_results = urllib2.urlopen(request).read()
|
||||
@ -836,13 +923,13 @@ class YoutubeIE(InfoExtractor):
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
|
||||
return
|
||||
|
||||
|
||||
# Confirm age
|
||||
age_form = {
|
||||
'next_url': '/',
|
||||
'action_confirm': 'Confirm',
|
||||
}
|
||||
request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
|
||||
request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
|
||||
try:
|
||||
self.report_age_confirmation()
|
||||
age_results = urllib2.urlopen(request).read()
|
||||
@ -860,7 +947,7 @@ class YoutubeIE(InfoExtractor):
|
||||
|
||||
# Get video webpage
|
||||
self.report_video_webpage_download(video_id)
|
||||
request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id, None, std_headers)
|
||||
request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
|
||||
try:
|
||||
video_webpage = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
@ -879,7 +966,7 @@ class YoutubeIE(InfoExtractor):
|
||||
for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
|
||||
video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
|
||||
% (video_id, el_type))
|
||||
request = urllib2.Request(video_info_url, None, std_headers)
|
||||
request = urllib2.Request(video_info_url)
|
||||
try:
|
||||
video_info_webpage = urllib2.urlopen(request).read()
|
||||
video_info = parse_qs(video_info_webpage)
|
||||
@ -1002,7 +1089,7 @@ class YoutubeIE(InfoExtractor):
|
||||
'player_url': player_url,
|
||||
})
|
||||
except UnavailableVideoError, err:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class MetacafeIE(InfoExtractor):
|
||||
@ -1028,18 +1115,18 @@ class MetacafeIE(InfoExtractor):
|
||||
def report_age_confirmation(self):
|
||||
"""Report attempt to confirm age."""
|
||||
self._downloader.to_screen(u'[metacafe] Confirming age')
|
||||
|
||||
|
||||
def report_download_webpage(self, video_id):
|
||||
"""Report webpage download."""
|
||||
self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
|
||||
|
||||
|
||||
def report_extraction(self, video_id):
|
||||
"""Report information extraction."""
|
||||
self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
|
||||
|
||||
def _real_initialize(self):
|
||||
# Retrieve disclaimer
|
||||
request = urllib2.Request(self._DISCLAIMER, None, std_headers)
|
||||
request = urllib2.Request(self._DISCLAIMER)
|
||||
try:
|
||||
self.report_disclaimer()
|
||||
disclaimer = urllib2.urlopen(request).read()
|
||||
@ -1052,14 +1139,14 @@ class MetacafeIE(InfoExtractor):
|
||||
'filters': '0',
|
||||
'submit': "Continue - I'm over 18",
|
||||
}
|
||||
request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
|
||||
request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
|
||||
try:
|
||||
self.report_age_confirmation()
|
||||
disclaimer = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
|
||||
return
|
||||
|
||||
|
||||
def _real_extract(self, url):
|
||||
# Extract id and simplified title from URL
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
@ -1095,7 +1182,7 @@ class MetacafeIE(InfoExtractor):
|
||||
if mobj is not None:
|
||||
mediaURL = urllib.unquote(mobj.group(1))
|
||||
video_extension = mediaURL[-3:]
|
||||
|
||||
|
||||
# Extract gdaKey if available
|
||||
mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
|
||||
if mobj is None:
|
||||
@ -1147,7 +1234,7 @@ class MetacafeIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class DailymotionIE(InfoExtractor):
|
||||
@ -1165,7 +1252,7 @@ class DailymotionIE(InfoExtractor):
|
||||
def report_download_webpage(self, video_id):
|
||||
"""Report webpage download."""
|
||||
self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
|
||||
|
||||
|
||||
def report_extraction(self, video_id):
|
||||
"""Report information extraction."""
|
||||
self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
|
||||
@ -1216,7 +1303,7 @@ class DailymotionIE(InfoExtractor):
|
||||
video_title = mobj.group(1).decode('utf-8')
|
||||
video_title = sanitize_title(video_title)
|
||||
|
||||
mobj = re.search(r'(?im)<div class="dmco_html owner">.*?<a class="name" href="/.+?">(.+?)</a>', webpage)
|
||||
mobj = re.search(r'(?im)<Attribute name="owner">(.+?)</Attribute>', webpage)
|
||||
if mobj is None:
|
||||
self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
|
||||
return
|
||||
@ -1236,7 +1323,7 @@ class DailymotionIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
class GoogleIE(InfoExtractor):
|
||||
"""Information extractor for video.google.com."""
|
||||
@ -1346,7 +1433,7 @@ class GoogleIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class PhotobucketIE(InfoExtractor):
|
||||
@ -1428,7 +1515,7 @@ class PhotobucketIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class YahooIE(InfoExtractor):
|
||||
@ -1586,7 +1673,7 @@ class YahooIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class GenericIE(InfoExtractor):
|
||||
@ -1687,7 +1774,7 @@ class GenericIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError, err:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class YoutubeSearchIE(InfoExtractor):
|
||||
@ -1702,7 +1789,7 @@ class YoutubeSearchIE(InfoExtractor):
|
||||
def __init__(self, youtube_ie, downloader=None):
|
||||
InfoExtractor.__init__(self, downloader)
|
||||
self._youtube_ie = youtube_ie
|
||||
|
||||
|
||||
@staticmethod
|
||||
def suitable(url):
|
||||
return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
|
||||
@ -1714,7 +1801,7 @@ class YoutubeSearchIE(InfoExtractor):
|
||||
|
||||
def _real_initialize(self):
|
||||
self._youtube_ie.initialize()
|
||||
|
||||
|
||||
def _real_extract(self, query):
|
||||
mobj = re.match(self._VALID_QUERY, query)
|
||||
if mobj is None:
|
||||
@ -1755,7 +1842,7 @@ class YoutubeSearchIE(InfoExtractor):
|
||||
while True:
|
||||
self.report_download_page(query, pagenum)
|
||||
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
|
||||
request = urllib2.Request(result_url, None, std_headers)
|
||||
request = urllib2.Request(result_url)
|
||||
try:
|
||||
page = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
@ -1793,7 +1880,7 @@ class GoogleSearchIE(InfoExtractor):
|
||||
def __init__(self, google_ie, downloader=None):
|
||||
InfoExtractor.__init__(self, downloader)
|
||||
self._google_ie = google_ie
|
||||
|
||||
|
||||
@staticmethod
|
||||
def suitable(url):
|
||||
return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
|
||||
@ -1805,7 +1892,7 @@ class GoogleSearchIE(InfoExtractor):
|
||||
|
||||
def _real_initialize(self):
|
||||
self._google_ie.initialize()
|
||||
|
||||
|
||||
def _real_extract(self, query):
|
||||
mobj = re.match(self._VALID_QUERY, query)
|
||||
if mobj is None:
|
||||
@ -1846,7 +1933,7 @@ class GoogleSearchIE(InfoExtractor):
|
||||
while True:
|
||||
self.report_download_page(query, pagenum)
|
||||
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
|
||||
request = urllib2.Request(result_url, None, std_headers)
|
||||
request = urllib2.Request(result_url)
|
||||
try:
|
||||
page = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
@ -1884,7 +1971,7 @@ class YahooSearchIE(InfoExtractor):
|
||||
def __init__(self, yahoo_ie, downloader=None):
|
||||
InfoExtractor.__init__(self, downloader)
|
||||
self._yahoo_ie = yahoo_ie
|
||||
|
||||
|
||||
@staticmethod
|
||||
def suitable(url):
|
||||
return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
|
||||
@ -1896,7 +1983,7 @@ class YahooSearchIE(InfoExtractor):
|
||||
|
||||
def _real_initialize(self):
|
||||
self._yahoo_ie.initialize()
|
||||
|
||||
|
||||
def _real_extract(self, query):
|
||||
mobj = re.match(self._VALID_QUERY, query)
|
||||
if mobj is None:
|
||||
@ -1937,7 +2024,7 @@ class YahooSearchIE(InfoExtractor):
|
||||
while True:
|
||||
self.report_download_page(query, pagenum)
|
||||
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
|
||||
request = urllib2.Request(result_url, None, std_headers)
|
||||
request = urllib2.Request(result_url)
|
||||
try:
|
||||
page = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
@ -1966,7 +2053,7 @@ class YahooSearchIE(InfoExtractor):
|
||||
class YoutubePlaylistIE(InfoExtractor):
|
||||
"""Information Extractor for YouTube playlists."""
|
||||
|
||||
_VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/)([^&]+).*'
|
||||
_VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/|p/)([^&]+).*'
|
||||
_TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
|
||||
_VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
|
||||
_MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
|
||||
@ -1975,7 +2062,7 @@ class YoutubePlaylistIE(InfoExtractor):
|
||||
def __init__(self, youtube_ie, downloader=None):
|
||||
InfoExtractor.__init__(self, downloader)
|
||||
self._youtube_ie = youtube_ie
|
||||
|
||||
|
||||
@staticmethod
|
||||
def suitable(url):
|
||||
return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
|
||||
@ -1986,7 +2073,7 @@ class YoutubePlaylistIE(InfoExtractor):
|
||||
|
||||
def _real_initialize(self):
|
||||
self._youtube_ie.initialize()
|
||||
|
||||
|
||||
def _real_extract(self, url):
|
||||
# Extract playlist id
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
@ -2001,7 +2088,7 @@ class YoutubePlaylistIE(InfoExtractor):
|
||||
|
||||
while True:
|
||||
self.report_download_page(playlist_id, pagenum)
|
||||
request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
|
||||
request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum))
|
||||
try:
|
||||
page = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
@ -2040,7 +2127,7 @@ class YoutubeUserIE(InfoExtractor):
|
||||
def __init__(self, youtube_ie, downloader=None):
|
||||
InfoExtractor.__init__(self, downloader)
|
||||
self._youtube_ie = youtube_ie
|
||||
|
||||
|
||||
@staticmethod
|
||||
def suitable(url):
|
||||
return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
|
||||
@ -2051,7 +2138,7 @@ class YoutubeUserIE(InfoExtractor):
|
||||
|
||||
def _real_initialize(self):
|
||||
self._youtube_ie.initialize()
|
||||
|
||||
|
||||
def _real_extract(self, url):
|
||||
# Extract username
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
@ -2137,7 +2224,7 @@ class DepositFilesIE(InfoExtractor):
|
||||
|
||||
# Retrieve file webpage with 'Free download' button pressed
|
||||
free_download_indication = { 'gateway_result' : '1' }
|
||||
request = urllib2.Request(url, urllib.urlencode(free_download_indication), std_headers)
|
||||
request = urllib2.Request(url, urllib.urlencode(free_download_indication))
|
||||
try:
|
||||
self.report_download_webpage(file_id)
|
||||
webpage = urllib2.urlopen(request).read()
|
||||
@ -2208,7 +2295,7 @@ class PostProcessor(object):
|
||||
def set_downloader(self, downloader):
|
||||
"""Sets the downloader for this PP."""
|
||||
self._downloader = downloader
|
||||
|
||||
|
||||
def run(self, information):
|
||||
"""Run the PostProcessor.
|
||||
|
||||
@ -2228,7 +2315,7 @@ class PostProcessor(object):
|
||||
it was called from.
|
||||
"""
|
||||
return information # by default, do nothing
|
||||
|
||||
|
||||
### MAIN PROGRAM ###
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
@ -2236,20 +2323,26 @@ if __name__ == '__main__':
|
||||
import getpass
|
||||
import optparse
|
||||
|
||||
# Function to update the program file with the latest version from bitbucket.org
|
||||
# Function to update the program file with the latest version from the repository.
|
||||
def update_self(downloader, filename):
|
||||
# Note: downloader only used for options
|
||||
if not os.access (filename, os.W_OK):
|
||||
if not os.access(filename, os.W_OK):
|
||||
sys.exit('ERROR: no write permissions on %s' % filename)
|
||||
|
||||
downloader.to_screen('Updating to latest stable version...')
|
||||
latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION'
|
||||
latest_version = urllib.urlopen(latest_url).read().strip()
|
||||
prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
|
||||
newcontent = urllib.urlopen(prog_url).read()
|
||||
stream = open(filename, 'w')
|
||||
stream.write(newcontent)
|
||||
stream.close()
|
||||
try:
|
||||
latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION'
|
||||
latest_version = urllib.urlopen(latest_url).read().strip()
|
||||
prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
|
||||
newcontent = urllib.urlopen(prog_url).read()
|
||||
except (IOError, OSError), err:
|
||||
sys.exit('ERROR: unable to download latest version')
|
||||
try:
|
||||
stream = open(filename, 'w')
|
||||
stream.write(newcontent)
|
||||
stream.close()
|
||||
except (IOError, OSError), err:
|
||||
sys.exit('ERROR: unable to overwrite current version')
|
||||
downloader.to_screen('Updated to version %s' % latest_version)
|
||||
|
||||
# Parse command line
|
||||
@ -2275,6 +2368,8 @@ if __name__ == '__main__':
|
||||
dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
|
||||
parser.add_option('--playlist-end',
|
||||
dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
|
||||
parser.add_option('--dump-user-agent',
|
||||
action='store_true', dest='dump_user_agent', help='display the current browser identification', default=False)
|
||||
|
||||
authentication = optparse.OptionGroup(parser, 'Authentication Options')
|
||||
authentication.add_option('-u', '--username',
|
||||
@ -2309,6 +2404,8 @@ if __name__ == '__main__':
|
||||
action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False)
|
||||
verbosity.add_option('--no-progress',
|
||||
action='store_true', dest='noprogress', help='do not print progress bar', default=False)
|
||||
verbosity.add_option('--console-title',
|
||||
action='store_true', dest='consoletitle', help='display progress in console titlebar', default=False)
|
||||
parser.add_option_group(verbosity)
|
||||
|
||||
filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
|
||||
@ -2328,6 +2425,8 @@ if __name__ == '__main__':
|
||||
action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
|
||||
filesystem.add_option('--cookies',
|
||||
dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
|
||||
filesystem.add_option('--no-part',
|
||||
action='store_true', dest='nopart', help='do not use .part files', default=False)
|
||||
parser.add_option_group(filesystem)
|
||||
|
||||
(opts, args) = parser.parse_args()
|
||||
@ -2343,10 +2442,14 @@ if __name__ == '__main__':
|
||||
except (IOError, OSError), err:
|
||||
sys.exit(u'ERROR: unable to open cookie file')
|
||||
|
||||
# Dump user agent
|
||||
if opts.dump_user_agent:
|
||||
print std_headers['User-Agent']
|
||||
sys.exit(0)
|
||||
|
||||
# General configuration
|
||||
cookie_processor = urllib2.HTTPCookieProcessor(jar)
|
||||
urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
|
||||
urllib2.install_opener(urllib2.build_opener(cookie_processor))
|
||||
urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler()))
|
||||
socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
|
||||
|
||||
# Batch file verification
|
||||
@ -2445,6 +2548,8 @@ if __name__ == '__main__':
|
||||
'playliststart': opts.playliststart,
|
||||
'playlistend': opts.playlistend,
|
||||
'logtostderr': opts.outtmpl == '-',
|
||||
'consoletitle': opts.consoletitle,
|
||||
'nopart': opts.nopart,
|
||||
})
|
||||
fd.add_info_extractor(youtube_search_ie)
|
||||
fd.add_info_extractor(youtube_pl_ie)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user