diff --git a/README.md b/README.md index ccd94b2dc..471cc8550 100644 --- a/README.md +++ b/README.md @@ -191,9 +191,9 @@ which means you can modify it, redistribute it or use it however you like. preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", - "bestaudio", "worst", and "worstaudio". By - default, youtube-dl will pick the best - quality. + "bestvideo", "bestaudio", "worst", + "worstvideo" and "worstaudio". By default, + youtube-dl will pick the best quality. --all-formats download all available video formats --prefer-free-formats prefer free video formats unless a specific one is requested diff --git a/devscripts/release.sh b/devscripts/release.sh index 72e708c7f..aa3119c42 100755 --- a/devscripts/release.sh +++ b/devscripts/release.sh @@ -70,7 +70,7 @@ RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz" git checkout HEAD -- youtube-dl youtube-dl.exe /bin/echo -e "\n### Signing and uploading the new binaries to yt-dl.org ..." -for f in $RELEASE_FILES; do gpg --detach-sig "build/$version/$f"; done +for f in $RELEASE_FILES; do gpg --passphrase-repeat 5 --detach-sig "build/$version/$f"; done scp -r "build/$version" ytdl@yt-dl.org:html/tmp/ ssh ytdl@yt-dl.org "mv html/tmp/$version html/downloads/" ssh ytdl@yt-dl.org "sh html/update_latest.sh $version" @@ -97,7 +97,7 @@ rm -rf build make pypi-files echo "Uploading to PyPi ..." -python setup.py sdist upload +python setup.py sdist bdist_wheel upload make clean /bin/echo -e "\n### DONE!" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..e57d130e3 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[wheel] +universal = True diff --git a/test/helper.py b/test/helper.py index b1f421ac5..17de951c5 100644 --- a/test/helper.py +++ b/test/helper.py @@ -71,7 +71,7 @@ class FakeYDL(YoutubeDL): old_report_warning(message) self.report_warning = types.MethodType(report_warning, self) -def get_testcases(): +def gettestcases(): for ie in youtube_dl.extractor.gen_extractors(): t = getattr(ie, '_TEST', None) if t: diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 047e84f19..3ce0b910a 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -9,7 +9,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import get_testcases +from test.helper import gettestcases from youtube_dl.extractor import ( FacebookIE, @@ -105,7 +105,7 @@ class TestAllURLsMatching(unittest.TestCase): def test_no_duplicates(self): ies = gen_extractors() - for tc in get_testcases(): + for tc in gettestcases(): url = tc['url'] for ie in ies: if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'): diff --git a/test/test_download.py b/test/test_download.py index ca8c82f71..8fccdaf9e 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -8,7 +8,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import ( get_params, - get_testcases, + gettestcases, try_rm, md5, report_warning @@ -51,7 +51,7 @@ def _file_md5(fn): with open(fn, 'rb') as f: return hashlib.md5(f.read()).hexdigest() -defs = get_testcases() +defs = gettestcases() class TestDownload(unittest.TestCase): diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 0e9504c14..07c9074fe 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -56,7 +56,6 @@ __authors__ = ( __license__ = 'Public Domain' import codecs -import getpass import io import locale import optparse @@ -68,6 +67,7 @@ import sys from .utils import ( + compat_getpass, compat_print, DateRange, decodeOption, @@ -611,7 +611,7 @@ def _real_main(argv=None): if opts.usetitle and opts.useid: parser.error(u'using title conflicts with using video ID') if opts.username is not None and opts.password is None: - opts.password = getpass.getpass(u'Type account password and press return:') + opts.password = compat_getpass(u'Type account password and press [Return]: ') if opts.ratelimit is not None: numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) if numeric_limit is None: diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 313414e7d..bbdb04069 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -196,6 +196,7 @@ from .rutube import ( RutubeMovieIE, RutubePersonIE, ) +from .rutv import RUTVIE from .savefrom import SaveFromIE from .servingsys import ServingSysIE from .sina import SinaIE @@ -251,8 +252,8 @@ from .ustream import UstreamIE, UstreamChannelIE from .vbox7 import Vbox7IE from .veehd import VeeHDIE from .veoh import VeohIE +from .vesti import VestiIE from .vevo import VevoIE -from .vgtrk import VGTRKIE from .vice import ViceIE from .viddler import ViddlerIE from .videobam import VideoBamIE diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py index ed3986f31..d50fcdbdb 100644 --- a/youtube_dl/extractor/comedycentral.py +++ b/youtube_dl/extractor/comedycentral.py @@ -14,7 +14,7 @@ from ..utils import ( class ComedyCentralIE(MTVServicesInfoExtractor): - _VALID_URL = r'''(?x)https?://(?:www\.)?comedycentral\.com/ + _VALID_URL = r'''(?x)https?://(?:www\.)?(comedycentral|cc)\.com/ (video-clips|episodes|cc-studios|video-collections) /(?P.*)''' _FEED_URL = 'http://comedycentral.com/feeds/mrss/' diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index afb08f76a..02f810956 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -24,6 +24,7 @@ from ..utils import ( ) from .brightcove import BrightcoveIE from .ooyala import OoyalaIE +from .rutv import RUTVIE class GenericIE(InfoExtractor): @@ -143,8 +144,22 @@ class GenericIE(InfoExtractor): 'ext': 'mp4', 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama', 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.', - } + }, }, + # RUTV embed + { + 'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html', + 'info_dict': { + 'id': '776940', + 'ext': 'mp4', + 'title': 'Охотское море стало целиком российским', + 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43', + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + } ] def report_download_webpage(self, video_id): @@ -170,9 +185,14 @@ class GenericIE(InfoExtractor): newurl = newurl.replace(' ', '%20') newheaders = dict((k,v) for k,v in req.headers.items() if k.lower() not in ("content-length", "content-type")) + try: + # This function was deprecated in python 3.3 and removed in 3.4 + origin_req_host = req.get_origin_req_host() + except AttributeError: + origin_req_host = req.origin_req_host return HEADRequest(newurl, headers=newheaders, - origin_req_host=req.get_origin_req_host(), + origin_req_host=origin_req_host, unverifiable=True) else: raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp) @@ -459,6 +479,11 @@ class GenericIE(InfoExtractor): return self.playlist_result( urlrs, playlist_id=video_id, playlist_title=video_title) + # Look for embedded RUTV player + rutv_url = RUTVIE._extract_url(webpage) + if rutv_url: + return self.url_result(rutv_url, 'RUTV') + # Start with something easy: JW Player in SWFObject mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) if mobj is None: diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py index 2a29e6072..d1defd363 100644 --- a/youtube_dl/extractor/iprima.py +++ b/youtube_dl/extractor/iprima.py @@ -48,7 +48,7 @@ class IPrimaIE(InfoExtractor): webpage = self._download_webpage(url, video_id) - if re.search(r'Nemáte oprávnění přistupovat na tuto stránku.\s*</div>', webpage): + if re.search(r'Nemáte oprávnění přistupovat na tuto stránku\.\s*</div>', webpage): raise ExtractorError( '%s said: You do not have permission to access this page' % self.IE_NAME, expected=True) diff --git a/youtube_dl/extractor/vgtrk.py b/youtube_dl/extractor/rutv.py similarity index 50% rename from youtube_dl/extractor/vgtrk.py rename to youtube_dl/extractor/rutv.py index 429b8bc72..5c38cbc02 100644 --- a/youtube_dl/extractor/vgtrk.py +++ b/youtube_dl/extractor/rutv.py @@ -10,141 +10,13 @@ from ..utils import ( ) -class VGTRKIE(InfoExtractor): - IE_DESC = 'ВГТРК' - _VALID_URL = r'http://(?:.+?\.)?(?:vesti\.ru|russia2?\.tv|tvkultura\.ru|rutv\.ru)/(?P<id>.+)' +class RUTVIE(InfoExtractor): + IE_DESC = 'RUTV.RU' + _VALID_URL = r'https?://player\.(?:rutv\.ru|vgtrk\.com)/(?:flash2v/container\.swf\?id=|iframe/(?P<type>swf|video|live)/id/)(?P<id>\d+)' _TESTS = [ { - 'url': 'http://www.vesti.ru/videos?vid=575582&cid=1', - 'info_dict': { - 'id': '765035', - 'ext': 'mp4', - 'title': 'Вести.net: биткоины в России не являются законными', - 'description': 'md5:d4bb3859dc1177b28a94c5014c35a36b', - 'duration': 302, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://www.vesti.ru/doc.html?id=1349233', - 'info_dict': { - 'id': '773865', - 'ext': 'mp4', - 'title': 'Участники митинга штурмуют Донецкую областную администрацию', - 'description': 'md5:1a160e98b3195379b4c849f2f4958009', - 'duration': 210, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://www.vesti.ru/only_video.html?vid=576180', - 'info_dict': { - 'id': '766048', - 'ext': 'mp4', - 'title': 'США заморозило, Британию затопило', - 'description': 'md5:f0ed0695ec05aed27c56a70a58dc4cc1', - 'duration': 87, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://hitech.vesti.ru/news/view/id/4000', - 'info_dict': { - 'id': '766888', - 'ext': 'mp4', - 'title': 'Вести.net: интернет-гиганты начали перетягивание программных "одеял"', - 'description': 'md5:65ddd47f9830c4f42ed6475f8730c995', - 'duration': 279, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://sochi2014.vesti.ru/video/index/video_id/766403', - 'info_dict': { - 'id': '766403', - 'ext': 'mp4', - 'title': 'XXII зимние Олимпийские игры. Российские хоккеисты стартовали на Олимпиаде с победы', - 'description': 'md5:55805dfd35763a890ff50fa9e35e31b3', - 'duration': 271, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - 'skip': 'Blocked outside Russia', - }, - { - 'url': 'http://sochi2014.vesti.ru/live/play/live_id/301', - 'info_dict': { - 'id': '51499', - 'ext': 'flv', - 'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ', - 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c', - }, - 'params': { - # rtmp download - 'skip_download': True, - }, - 'skip': 'Translation has finished' - }, - { - 'url': 'http://russia.tv/video/show/brand_id/5169/episode_id/970443/video_id/975648', - 'info_dict': { - 'id': '771852', - 'ext': 'mp4', - 'title': 'Прямой эфир. Жертвы загадочной болезни: смерть от старости в 17 лет', - 'description': 'md5:b81c8c55247a4bd996b43ce17395b2d8', - 'duration': 3096, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://russia.tv/brand/show/brand_id/57638', - 'info_dict': { - 'id': '774016', - 'ext': 'mp4', - 'title': 'Чужой в семье Сталина', - 'description': '', - 'duration': 2539, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://2.russia.tv/video/show/brand_id/48863/episode_id/972920/video_id/978667/viewtype/picture', - 'info_dict': { - 'id': '775081', - 'ext': 'mp4', - 'title': 'XXII зимние Олимпийские игры. Россияне заняли весь пьедестал в лыжных гонках', - 'description': 'md5:15d3741dd8d04b203fbc031c6a47fb0f', - 'duration': 101, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - 'skip': 'Blocked outside Russia', - }, - { - 'url': 'http://tvkultura.ru/video/show/brand_id/31724/episode_id/972347/video_id/978186', + 'url': 'http://player.rutv.ru/flash2v/container.swf?id=774471&sid=kultura&fbv=true&isPlay=true&ssl=false&i=560&acc_video_id=episode_id/972347/video_id/978186/brand_id/31724', 'info_dict': { 'id': '774471', 'ext': 'mp4', @@ -158,64 +30,97 @@ class VGTRKIE(InfoExtractor): }, }, { - 'url': 'http://rutv.ru/brand/show/id/6792/channel/75', + 'url': 'https://player.vgtrk.com/flash2v/container.swf?id=774016&sid=russiatv&fbv=true&isPlay=true&ssl=false&i=560&acc_video_id=episode_id/972098/video_id/977760/brand_id/57638', 'info_dict': { - 'id': '125521', + 'id': '774016', 'ext': 'mp4', - 'title': 'Грустная дама червей. Х/ф', + 'title': 'Чужой в семье Сталина', 'description': '', - 'duration': 4882, + 'duration': 2539, }, 'params': { # m3u8 download 'skip_download': True, }, }, + { + 'url': 'http://player.rutv.ru/iframe/swf/id/766888/sid/hitech/?acc_video_id=4000', + 'info_dict': { + 'id': '766888', + 'ext': 'mp4', + 'title': 'Вести.net: интернет-гиганты начали перетягивание программных "одеял"', + 'description': 'md5:65ddd47f9830c4f42ed6475f8730c995', + 'duration': 279, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://player.rutv.ru/iframe/video/id/771852/start_zoom/true/showZoomBtn/false/sid/russiatv/?acc_video_id=episode_id/970443/video_id/975648/brand_id/5169', + 'info_dict': { + 'id': '771852', + 'ext': 'mp4', + 'title': 'Прямой эфир. Жертвы загадочной болезни: смерть от старости в 17 лет', + 'description': 'md5:b81c8c55247a4bd996b43ce17395b2d8', + 'duration': 3096, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://player.rutv.ru/iframe/live/id/51499/showZoomBtn/false/isPlay/true/sid/sochi2014', + 'info_dict': { + 'id': '51499', + 'ext': 'flv', + 'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ', + 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c', + }, + 'params': { + # rtmp download + 'skip_download': True, + }, + 'skip': 'Translation has finished', + }, ] + @classmethod + def _extract_url(cls, webpage): + mobj = re.search( + r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.rutv\.ru/iframe/(?:swf|video|live)/id/.+?)\1', webpage) + if mobj: + return mobj.group('url') + + mobj = re.search( + r'<meta[^>]+?property=(["\'])og:video\1[^>]+?content=(["\'])(?P<url>http://player\.(?:rutv\.ru|vgtrk\.com)/flash2v/container\.swf\?id=.+?\2)', + webpage) + if mobj: + return mobj.group('url') + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') + video_type = mobj.group('type') - page = self._download_webpage(url, video_id, 'Downloading page') - - mobj = re.search( - r'<meta property="og:video" content="http://www\.vesti\.ru/i/flvplayer_videoHost\.swf\?vid=(?P<id>\d+)', - page) - if mobj: - video_id = mobj.group('id') - page = self._download_webpage('http://www.vesti.ru/only_video.html?vid=%s' % video_id, video_id, - 'Downloading video page') - - mobj = re.search( - r'<meta property="og:video" content="http://player\.rutv\.ru/flash2v/container\.swf\?id=(?P<id>\d+)', page) - if mobj: + if not video_type or video_type == 'swf': video_type = 'video' - video_id = mobj.group('id') - else: - mobj = re.search( - r'<iframe.+?src="http://player\.rutv\.ru/iframe/(?P<type>[^/]+)/id/(?P<id>\d+)[^"]*".*?></iframe>', - page) - - if not mobj: - raise ExtractorError('No media found', expected=True) - - video_type = mobj.group('type') - video_id = mobj.group('id') json_data = self._download_json( 'http://player.rutv.ru/iframe/%splay/id/%s' % ('live-' if video_type == 'live' else '', video_id), video_id, 'Downloading JSON') if json_data['errors']: - raise ExtractorError('vesti returned error: %s' % json_data['errors'], expected=True) + raise ExtractorError('%s said: %s' % (self.IE_NAME, json_data['errors']), expected=True) playlist = json_data['data']['playlist'] medialist = playlist['medialist'] media = medialist[0] if media['errors']: - raise ExtractorError('vesti returned error: %s' % media['errors'], expected=True) + raise ExtractorError('%s said: %s' % (self.IE_NAME, media['errors']), expected=True) view_count = playlist.get('count_views') priority_transport = playlist['priority_transport'] diff --git a/youtube_dl/extractor/vesti.py b/youtube_dl/extractor/vesti.py new file mode 100644 index 000000000..27f9acb67 --- /dev/null +++ b/youtube_dl/extractor/vesti.py @@ -0,0 +1,121 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ExtractorError +from .rutv import RUTVIE + + +class VestiIE(InfoExtractor): + IE_DESC = 'Вести.Ru' + _VALID_URL = r'http://(?:.+?\.)?vesti\.ru/(?P<id>.+)' + + _TESTS = [ + { + 'url': 'http://www.vesti.ru/videos?vid=575582&cid=1', + 'info_dict': { + 'id': '765035', + 'ext': 'mp4', + 'title': 'Вести.net: биткоины в России не являются законными', + 'description': 'md5:d4bb3859dc1177b28a94c5014c35a36b', + 'duration': 302, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://www.vesti.ru/doc.html?id=1349233', + 'info_dict': { + 'id': '773865', + 'ext': 'mp4', + 'title': 'Участники митинга штурмуют Донецкую областную администрацию', + 'description': 'md5:1a160e98b3195379b4c849f2f4958009', + 'duration': 210, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://www.vesti.ru/only_video.html?vid=576180', + 'info_dict': { + 'id': '766048', + 'ext': 'mp4', + 'title': 'США заморозило, Британию затопило', + 'description': 'md5:f0ed0695ec05aed27c56a70a58dc4cc1', + 'duration': 87, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://hitech.vesti.ru/news/view/id/4000', + 'info_dict': { + 'id': '766888', + 'ext': 'mp4', + 'title': 'Вести.net: интернет-гиганты начали перетягивание программных "одеял"', + 'description': 'md5:65ddd47f9830c4f42ed6475f8730c995', + 'duration': 279, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://sochi2014.vesti.ru/video/index/video_id/766403', + 'info_dict': { + 'id': '766403', + 'ext': 'mp4', + 'title': 'XXII зимние Олимпийские игры. Российские хоккеисты стартовали на Олимпиаде с победы', + 'description': 'md5:55805dfd35763a890ff50fa9e35e31b3', + 'duration': 271, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + 'skip': 'Blocked outside Russia', + }, + { + 'url': 'http://sochi2014.vesti.ru/live/play/live_id/301', + 'info_dict': { + 'id': '51499', + 'ext': 'flv', + 'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ', + 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c', + }, + 'params': { + # rtmp download + 'skip_download': True, + }, + 'skip': 'Translation has finished' + }, + ] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + page = self._download_webpage(url, video_id, 'Downloading page') + + mobj = re.search( + r'<meta[^>]+?property="og:video"[^>]+?content="http://www\.vesti\.ru/i/flvplayer_videoHost\.swf\?vid=(?P<id>\d+)', + page) + if mobj: + video_id = mobj.group('id') + page = self._download_webpage('http://www.vesti.ru/only_video.html?vid=%s' % video_id, video_id, + 'Downloading video page') + + rutv_url = RUTVIE._extract_url(page) + if rutv_url: + return self.url_result(rutv_url, 'RUTV') + + raise ExtractorError('No video found', expected=True) \ No newline at end of file diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 3cf29e63a..f60141bd1 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -6,6 +6,7 @@ import ctypes import datetime import email.utils import errno +import getpass import gzip import itertools import io @@ -1279,3 +1280,12 @@ def parse_xml(s): parser = xml.etree.ElementTree.XMLParser(target=TreeBuilder()) kwargs = {'parser': parser} if sys.version_info >= (2, 7) else {} return xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs) + + +if sys.version_info < (3, 0) and sys.platform == 'win32': + def compat_getpass(prompt, *args, **kwargs): + if isinstance(prompt, compat_str): + prompt = prompt.encode(preferredencoding()) + return getpass.getpass(prompt, *args, **kwargs) +else: + compat_getpass = getpass.getpass diff --git a/youtube_dl/version.py b/youtube_dl/version.py index c038225f7..cafe4b75c 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2014.03.12' +__version__ = '2014.03.18.1'