Akkariin/youtube-dl
1
0
Fork 0
mirror of https://github.com/l1ving/youtube-dl synced 2025-02-11 07:22:51 +08:00
Code Issues Releases Wiki Activity

Merge branch 'master' into use-other-downloaders

Browse Source
This commit is contained in:
Rogério Brito 2014-01-19 02:47:56 -02:00
commit c689f4b9c7
84 changed files with 2567 additions and 1202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -23,6 +23,8 @@ updates_key.pem
*.vtt *.vtt
*.flv *.flv
*.mp4 *.mp4
*.m4a
*.m4v
*.part *.part
test/testdata test/testdata
.tox .tox

57
README.md
View File

@ -34,10 +34,13 @@ which means you can modify it, redistribute it or use it however you like.
empty string (--proxy "") for direct connection empty string (--proxy "") for direct connection
--no-check-certificate Suppress HTTPS certificate validation. --no-check-certificate Suppress HTTPS certificate validation.
--cache-dir DIR Location in the filesystem where youtube-dl can --cache-dir DIR Location in the filesystem where youtube-dl can
store downloaded information permanently. By store some downloaded information permanently. By
default $XDG_CACHE_HOME/youtube-dl or ~/.cache default $XDG_CACHE_HOME/youtube-dl or ~/.cache
/youtube-dl . /youtube-dl . At the moment, only YouTube player
files (for videos with obfuscated signatures) are
cached, but that may change.
--no-cache-dir Disable filesystem caching --no-cache-dir Disable filesystem caching
--socket-timeout None Time to wait before giving up, in seconds
--bidi-workaround Work around terminals that lack bidirectional --bidi-workaround Work around terminals that lack bidirectional
text support. Requires bidiv or fribidi text support. Requires bidiv or fribidi
executable in PATH executable in PATH
@ -55,8 +58,10 @@ which means you can modify it, redistribute it or use it however you like.
--max-filesize SIZE Do not download any videos larger than SIZE (e.g. --max-filesize SIZE Do not download any videos larger than SIZE (e.g.
50k or 44.6m) 50k or 44.6m)
--date DATE download only videos uploaded in this date --date DATE download only videos uploaded in this date
--datebefore DATE download only videos uploaded before this date --datebefore DATE download only videos uploaded on or before this
--dateafter DATE download only videos uploaded after this date date (i.e. inclusive)
--dateafter DATE download only videos uploaded on or after this
date (i.e. inclusive)
--min-views COUNT Do not download any videos with less than COUNT --min-views COUNT Do not download any videos with less than COUNT
views views
--max-views COUNT Do not download any videos with more than COUNT --max-views COUNT Do not download any videos with more than COUNT
@ -88,13 +93,13 @@ which means you can modify it, redistribute it or use it however you like.
different, %(autonumber)s to get an automatically different, %(autonumber)s to get an automatically
incremented number, %(ext)s for the filename incremented number, %(ext)s for the filename
extension, %(format)s for the format description extension, %(format)s for the format description
(like "22 - 1280x720" or "HD"),%(format_id)s for (like "22 - 1280x720" or "HD"), %(format_id)s for
the unique id of the format (like Youtube's the unique id of the format (like Youtube's
itags: "137"),%(upload_date)s for the upload date itags: "137"), %(upload_date)s for the upload
(YYYYMMDD), %(extractor)s for the provider date (YYYYMMDD), %(extractor)s for the provider
(youtube, metacafe, etc), %(id)s for the video id (youtube, metacafe, etc), %(id)s for the video
, %(playlist)s for the playlist the video is in, id, %(playlist)s for the playlist the video is
%(playlist_index)s for the position in the in, %(playlist_index)s for the position in the
playlist and %% for a literal percent. Use - to playlist and %% for a literal percent. Use - to
output to stdout. Can also be used to download to output to stdout. Can also be used to download to
a different directory, for example with -o '/my/d a different directory, for example with -o '/my/d
@ -106,7 +111,7 @@ which means you can modify it, redistribute it or use it however you like.
avoid "&" and spaces in filenames avoid "&" and spaces in filenames
-a, --batch-file FILE file containing URLs to download ('-' for stdin) -a, --batch-file FILE file containing URLs to download ('-' for stdin)
--load-info FILE json file containing the video information --load-info FILE json file containing the video information
(created with the "--write-json" option (created with the "--write-json" option)
-w, --no-overwrites do not overwrite files -w, --no-overwrites do not overwrite files
-c, --continue force resume of partially downloaded files. By -c, --continue force resume of partially downloaded files. By
default, youtube-dl will resume downloads if default, youtube-dl will resume downloads if
@ -140,7 +145,7 @@ which means you can modify it, redistribute it or use it however you like.
--no-progress do not print progress bar --no-progress do not print progress bar
--console-title display progress in console titlebar --console-title display progress in console titlebar
-v, --verbose print various debugging information -v, --verbose print various debugging information
--dump-intermediate-pages print downloaded pages to debug problems(very --dump-intermediate-pages print downloaded pages to debug problems (very
verbose) verbose)
--write-pages Write downloaded intermediary pages to files in --write-pages Write downloaded intermediary pages to files in
the current directory to debug problems the current directory to debug problems
@ -153,8 +158,7 @@ which means you can modify it, redistribute it or use it however you like.
--prefer-free-formats prefer free video formats unless a specific one --prefer-free-formats prefer free video formats unless a specific one
is requested is requested
--max-quality FORMAT highest quality format to download --max-quality FORMAT highest quality format to download
-F, --list-formats list all available formats (currently youtube -F, --list-formats list all available formats
only)
## Subtitle Options: ## Subtitle Options:
--write-sub write subtitle file --write-sub write subtitle file
@ -172,7 +176,7 @@ which means you can modify it, redistribute it or use it however you like.
-u, --username USERNAME account username -u, --username USERNAME account username
-p, --password PASSWORD account password -p, --password PASSWORD account password
-n, --netrc use .netrc authentication data -n, --netrc use .netrc authentication data
--video-password PASSWORD video password (vimeo only) --video-password PASSWORD video password (vimeo, smotri)
## Post-processing Options: ## Post-processing Options:
-x, --extract-audio convert video files to audio-only files (requires -x, --extract-audio convert video files to audio-only files (requires
@ -190,7 +194,13 @@ which means you can modify it, redistribute it or use it however you like.
processed files are overwritten by default processed files are overwritten by default
--embed-subs embed subtitles in the video (only for mp4 --embed-subs embed subtitles in the video (only for mp4
videos) videos)
--add-metadata add metadata to the files --add-metadata write metadata to the video file
--xattrs write metadata to the video file's xattrs (using
dublin core and xdg standards)
--prefer-avconv Prefer avconv over ffmpeg for running the
postprocessors (default)
--prefer-ffmpeg Prefer ffmpeg over avconv for running the
postprocessors
# CONFIGURATION # CONFIGURATION
@ -229,9 +239,12 @@ Videos can be filtered by their upload date using the options `--date`, `--dateb
Examples: Examples:
$ youtube-dl --dateafter now-6months #will only download the videos uploaded in the last 6 months $ # Download only the videos uploaded in the last 6 months
$ youtube-dl --date 19700101 #will only download the videos uploaded in January 1, 1970 $ youtube-dl --dateafter now-6months
$ youtube-dl --dateafter 20000101 --datebefore 20100101 #will only download the videos uploaded between 2000 and 2010 $ # Download only the videos uploaded on January 1, 1970
$ youtube-dl --date 19700101
$ # will only download the videos uploaded in the 200x decade
$ youtube-dl --dateafter 20000101 --datebefore 20091231
# FAQ # FAQ
@ -310,7 +323,7 @@ Site support requests must contain an example URL. An example URL is a URL you m
### Are you using the latest version? ### Are you using the latest version?
Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. Ábout 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well. Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
### Is the issue already documented? ### Is the issue already documented?
@ -335,3 +348,7 @@ In particular, every site support request issue should only pertain to services
### Is anyone going to need the feature? ### Is anyone going to need the feature?
Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them. Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
### Is your question about youtube-dl?
It may sound strange, but some bug reports we receive are completely unrelated to youtube-dl and relate to a different or even the reporter's own application. Please make sure that you are actually using youtube-dl. If you are using a UI for youtube-dl, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for youtube-dl fails in some way you believe is related to youtube-dl, by all means, go ahead and report the bug.

2
devscripts/bash-completion.in
View File

@ -6,7 +6,7 @@ __youtube_dl()
prev="${COMP_WORDS[COMP_CWORD-1]}" prev="${COMP_WORDS[COMP_CWORD-1]}"
opts="{{flags}}" opts="{{flags}}"
keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory" keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory"
fileopts="-a|--batch-file|--download-archive|--cookies" fileopts="-a|--batch-file|--download-archive|--cookies|--load-info"
diropts="--cache-dir" diropts="--cache-dir"
if [[ ${prev} =~ ${fileopts} ]]; then if [[ ${prev} =~ ${fileopts} ]]; then

44
devscripts/check-porn.py
View File

@ -3,6 +3,9 @@
""" """
This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check
if we are not 'age_limit' tagging some porn site if we are not 'age_limit' tagging some porn site
A second approach implemented relies on a list of porn domains, to activate it
pass the list filename as the only argument
""" """
# Allow direct execution # Allow direct execution
@ -11,25 +14,42 @@ import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from test.helper import get_testcases from test.helper import get_testcases
from youtube_dl.utils import compat_urllib_parse_urlparse
from youtube_dl.utils import compat_urllib_request from youtube_dl.utils import compat_urllib_request
if len(sys.argv) > 1:
METHOD = 'LIST'
LIST = open(sys.argv[1]).read().decode('utf8').strip()
else:
METHOD = 'EURISTIC'
for test in get_testcases(): for test in get_testcases():
try: if METHOD == 'EURISTIC':
webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read() try:
except: webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
print('\nFail: {0}'.format(test['name'])) except:
continue print('\nFail: {0}'.format(test['name']))
continue
webpage = webpage.decode('utf8', 'replace') webpage = webpage.decode('utf8', 'replace')
if 'porn' in webpage.lower() and ('info_dict' not in test RESULT = 'porn' in webpage.lower()
or 'age_limit' not in test['info_dict']
or test['info_dict']['age_limit'] != 18): elif METHOD == 'LIST':
domain = compat_urllib_parse_urlparse(test['url']).netloc
if not domain:
print('\nFail: {0}'.format(test['name']))
continue
domain = '.'.join(domain.split('.')[-2:])
RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST)
if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict']
or test['info_dict']['age_limit'] != 18):
print('\nPotential missing age_limit check: {0}'.format(test['name'])) print('\nPotential missing age_limit check: {0}'.format(test['name']))
elif 'porn' not in webpage.lower() and ('info_dict' in test and elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict']
'age_limit' in test['info_dict'] and and test['info_dict']['age_limit'] == 18):
test['info_dict']['age_limit'] == 18):
print('\nPotential false negative: {0}'.format(test['name'])) print('\nPotential false negative: {0}'.format(test['name']))
else: else:

90
devscripts/gh-pages/update-feed.py
View File

@ -1,56 +1,76 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import datetime import datetime
import io
import json
import textwrap import textwrap
import json
atom_template=textwrap.dedent("""\ atom_template = textwrap.dedent("""\
<?xml version='1.0' encoding='utf-8'?> <?xml version="1.0" encoding="utf-8"?>
<atom:feed xmlns:atom="http://www.w3.org/2005/Atom"> <feed xmlns="http://www.w3.org/2005/Atom">
<atom:title>youtube-dl releases</atom:title> <link rel="self" href="http://rg3.github.io/youtube-dl/update/releases.atom" />
<atom:id>youtube-dl-updates-feed</atom:id> <title>youtube-dl releases</title>
<atom:updated>@TIMESTAMP@</atom:updated> <id>https://yt-dl.org/feed/youtube-dl-updates-feed</id>
@ENTRIES@ <updated>@TIMESTAMP@</updated>
</atom:feed>""") @ENTRIES@
</feed>""")
entry_template=textwrap.dedent(""" entry_template = textwrap.dedent("""
<atom:entry> <entry>
<atom:id>youtube-dl-@VERSION@</atom:id> <id>https://yt-dl.org/feed/youtube-dl-updates-feed/youtube-dl-@VERSION@</id>
<atom:title>New version @VERSION@</atom:title> <title>New version @VERSION@</title>
<atom:link href="http://rg3.github.io/youtube-dl" /> <link href="http://rg3.github.io/youtube-dl" />
<atom:content type="xhtml"> <content type="xhtml">
<div xmlns="http://www.w3.org/1999/xhtml"> <div xmlns="http://www.w3.org/1999/xhtml">
Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a> Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a>
</div> </div>
</atom:content> </content>
<atom:author> <author>
<atom:name>The youtube-dl maintainers</atom:name> <name>The youtube-dl maintainers</name>
</atom:author> </author>
<atom:updated>@TIMESTAMP@</atom:updated> <updated>@TIMESTAMP@</updated>
</atom:entry> </entry>
""") """)
now = datetime.datetime.now() now = datetime.datetime.now()
now_iso = now.isoformat() now_iso = now.isoformat() + 'Z'
atom_template = atom_template.replace('@TIMESTAMP@',now_iso) atom_template = atom_template.replace('@TIMESTAMP@', now_iso)
entries=[]
versions_info = json.load(open('update/versions.json')) versions_info = json.load(open('update/versions.json'))
versions = list(versions_info['versions'].keys()) versions = list(versions_info['versions'].keys())
versions.sort() versions.sort()
entries = []
for v in versions: for v in versions:
entry = entry_template.replace('@TIMESTAMP@',v.replace('.','-')) fields = v.split('.')
entry = entry.replace('@VERSION@',v) year, month, day = map(int, fields[:3])
entries.append(entry) faked = 0
patchlevel = 0
while True:
try:
datetime.date(year, month, day)
except ValueError:
day -= 1
faked += 1
assert day > 0
continue
break
if len(fields) >= 4:
try:
patchlevel = int(fields[3])
except ValueError:
patchlevel = 1
timestamp = '%04d-%02d-%02dT00:%02d:%02dZ' % (year, month, day, faked, patchlevel)
entry = entry_template.replace('@TIMESTAMP@', timestamp)
entry = entry.replace('@VERSION@', v)
entries.append(entry)
entries_str = textwrap.indent(''.join(entries), '\t') entries_str = textwrap.indent(''.join(entries), '\t')
atom_template = atom_template.replace('@ENTRIES@', entries_str) atom_template = atom_template.replace('@ENTRIES@', entries_str)
with open('update/releases.atom','w',encoding='utf-8') as atom_file: with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file:
atom_file.write(atom_template) atom_file.write(atom_template)

10
devscripts/make_readme.py
View File

@ -1,20 +1,24 @@
import io
import sys import sys
import re import re
README_FILE = 'README.md' README_FILE = 'README.md'
helptext = sys.stdin.read() helptext = sys.stdin.read()
with open(README_FILE) as f: if isinstance(helptext, bytes):
helptext = helptext.decode('utf-8')
with io.open(README_FILE, encoding='utf-8') as f:
oldreadme = f.read() oldreadme = f.read()
header = oldreadme[:oldreadme.index('# OPTIONS')] header = oldreadme[:oldreadme.index('# OPTIONS')]
footer = oldreadme[oldreadme.index('# CONFIGURATION'):] footer = oldreadme[oldreadme.index('# CONFIGURATION'):]
options = helptext[helptext.index(' General Options:')+19:] options = helptext[helptext.index(' General Options:') + 19:]
options = re.sub(r'^ (\w.+)$', r'## \1', options, flags=re.M) options = re.sub(r'^ (\w.+)$', r'## \1', options, flags=re.M)
options = '# OPTIONS\n' + options + '\n' options = '# OPTIONS\n' + options + '\n'
with open(README_FILE, 'w') as f: with io.open(README_FILE, 'w', encoding='utf-8') as f:
f.write(header) f.write(header)
f.write(options) f.write(options)
f.write(footer) f.write(footer)

2
devscripts/release.sh
View File

@ -24,6 +24,8 @@ if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.0
version="$1" version="$1"
if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
useless_files=$(find youtube_dl -type f -not -name '*.py')
if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $useless_files"; exit 1; fi
if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
/bin/echo -e "\n### First of all, testing..." /bin/echo -e "\n### First of all, testing..."

5
setup.py
View File

@ -71,7 +71,10 @@ setup(
author_email='ytdl@yt-dl.org', author_email='ytdl@yt-dl.org',
maintainer='Philipp Hagemeister', maintainer='Philipp Hagemeister',
maintainer_email='phihag@phihag.de', maintainer_email='phihag@phihag.de',
packages=['youtube_dl', 'youtube_dl.extractor', 'youtube_dl.downloader'], packages=[
'youtube_dl',
'youtube_dl.extractor', 'youtube_dl.downloader',
'youtube_dl.postprocessor'],
# Provokes warning on most systems (why?!) # Provokes warning on most systems (why?!)
# test_suite = 'nose.collector', # test_suite = 'nose.collector',

2
test/test_all_urls.py
View File

@ -113,6 +113,8 @@ class TestAllURLsMatching(unittest.TestCase):
def test_vimeo_matching(self): def test_vimeo_matching(self):
self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel']) self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel'])
self.assertMatch('http://vimeo.com/user7108434', ['vimeo:user']) self.assertMatch('http://vimeo.com/user7108434', ['vimeo:user'])
self.assertMatch('http://vimeo.com/user7108434/videos', ['vimeo:user'])
self.assertMatch('https://vimeo.com/user21297594/review/75524534/3c257a1b5d', ['vimeo:review'])
# https://github.com/rg3/youtube-dl/issues/1930 # https://github.com/rg3/youtube-dl/issues/1930
def test_soundcloud_not_matching_sets(self): def test_soundcloud_not_matching_sets(self):

2
test/test_download.py
View File

@ -148,7 +148,7 @@ def generator(test_case):
for key, value in info_dict.items() for key, value in info_dict.items()
if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location')) if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location'))
if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()): if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()):
sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=2) + u'\n') sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=4) + u'\n')
# Check for the presence of mandatory fields # Check for the presence of mandatory fields
for key in ('id', 'url', 'title', 'ext'): for key in ('id', 'url', 'title', 'ext'):

84
test/test_playlists.py
View File

@ -1,6 +1,7 @@
#!/usr/bin/env python #!/usr/bin/env python
# encoding: utf-8 # encoding: utf-8
from __future__ import unicode_literals
# Allow direct execution # Allow direct execution
import os import os
@ -28,7 +29,10 @@ from youtube_dl.extractor import (
BandcampAlbumIE, BandcampAlbumIE,
SmotriCommunityIE, SmotriCommunityIE,
SmotriUserIE, SmotriUserIE,
IviCompilationIE IviCompilationIE,
ImdbListIE,
KhanAcademyIE,
EveryonesMixtapeIE,
) )
@ -42,7 +46,7 @@ class TestPlaylists(unittest.TestCase):
ie = DailymotionPlaylistIE(dl) ie = DailymotionPlaylistIE(dl)
result = ie.extract('http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q') result = ie.extract('http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['title'], u'SPORT') self.assertEqual(result['title'], 'SPORT')
self.assertTrue(len(result['entries']) > 20) self.assertTrue(len(result['entries']) > 20)
def test_dailymotion_user(self): def test_dailymotion_user(self):
@ -50,7 +54,7 @@ class TestPlaylists(unittest.TestCase):
ie = DailymotionUserIE(dl) ie = DailymotionUserIE(dl)
result = ie.extract('http://www.dailymotion.com/user/generation-quoi/') result = ie.extract('http://www.dailymotion.com/user/generation-quoi/')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['title'], u'Génération Quoi') self.assertEqual(result['title'], 'Génération Quoi')
self.assertTrue(len(result['entries']) >= 26) self.assertTrue(len(result['entries']) >= 26)
def test_vimeo_channel(self): def test_vimeo_channel(self):
@ -58,7 +62,7 @@ class TestPlaylists(unittest.TestCase):
ie = VimeoChannelIE(dl) ie = VimeoChannelIE(dl)
result = ie.extract('http://vimeo.com/channels/tributes') result = ie.extract('http://vimeo.com/channels/tributes')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['title'], u'Vimeo Tributes') self.assertEqual(result['title'], 'Vimeo Tributes')
self.assertTrue(len(result['entries']) > 24) self.assertTrue(len(result['entries']) > 24)
def test_vimeo_user(self): def test_vimeo_user(self):
@ -66,7 +70,7 @@ class TestPlaylists(unittest.TestCase):
ie = VimeoUserIE(dl) ie = VimeoUserIE(dl)
result = ie.extract('http://vimeo.com/nkistudio/videos') result = ie.extract('http://vimeo.com/nkistudio/videos')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['title'], u'Nki') self.assertEqual(result['title'], 'Nki')
self.assertTrue(len(result['entries']) > 65) self.assertTrue(len(result['entries']) > 65)
def test_vimeo_album(self): def test_vimeo_album(self):
@ -74,7 +78,7 @@ class TestPlaylists(unittest.TestCase):
ie = VimeoAlbumIE(dl) ie = VimeoAlbumIE(dl)
result = ie.extract('http://vimeo.com/album/2632481') result = ie.extract('http://vimeo.com/album/2632481')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['title'], u'Staff Favorites: November 2013') self.assertEqual(result['title'], 'Staff Favorites: November 2013')
self.assertTrue(len(result['entries']) > 12) self.assertTrue(len(result['entries']) > 12)
def test_vimeo_groups(self): def test_vimeo_groups(self):
@ -82,7 +86,7 @@ class TestPlaylists(unittest.TestCase):
ie = VimeoGroupsIE(dl) ie = VimeoGroupsIE(dl)
result = ie.extract('http://vimeo.com/groups/rolexawards') result = ie.extract('http://vimeo.com/groups/rolexawards')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['title'], u'Rolex Awards for Enterprise') self.assertEqual(result['title'], 'Rolex Awards for Enterprise')
self.assertTrue(len(result['entries']) > 72) self.assertTrue(len(result['entries']) > 72)
def test_ustream_channel(self): def test_ustream_channel(self):
@ -90,7 +94,7 @@ class TestPlaylists(unittest.TestCase):
ie = UstreamChannelIE(dl) ie = UstreamChannelIE(dl)
result = ie.extract('http://www.ustream.tv/channel/young-americans-for-liberty') result = ie.extract('http://www.ustream.tv/channel/young-americans-for-liberty')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['id'], u'5124905') self.assertEqual(result['id'], '5124905')
self.assertTrue(len(result['entries']) >= 11) self.assertTrue(len(result['entries']) >= 11)
def test_soundcloud_set(self): def test_soundcloud_set(self):
@ -98,7 +102,7 @@ class TestPlaylists(unittest.TestCase):
ie = SoundcloudSetIE(dl) ie = SoundcloudSetIE(dl)
result = ie.extract('https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep') result = ie.extract('https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['title'], u'The Royal Concept EP') self.assertEqual(result['title'], 'The Royal Concept EP')
self.assertTrue(len(result['entries']) >= 6) self.assertTrue(len(result['entries']) >= 6)
def test_soundcloud_user(self): def test_soundcloud_user(self):
@ -106,7 +110,7 @@ class TestPlaylists(unittest.TestCase):
ie = SoundcloudUserIE(dl) ie = SoundcloudUserIE(dl)
result = ie.extract('https://soundcloud.com/the-concept-band') result = ie.extract('https://soundcloud.com/the-concept-band')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['id'], u'9615865') self.assertEqual(result['id'], '9615865')
self.assertTrue(len(result['entries']) >= 12) self.assertTrue(len(result['entries']) >= 12)
def test_livestream_event(self): def test_livestream_event(self):
@ -114,7 +118,7 @@ class TestPlaylists(unittest.TestCase):
ie = LivestreamIE(dl) ie = LivestreamIE(dl)
result = ie.extract('http://new.livestream.com/tedx/cityenglish') result = ie.extract('http://new.livestream.com/tedx/cityenglish')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['title'], u'TEDCity2.0 (English)') self.assertEqual(result['title'], 'TEDCity2.0 (English)')
self.assertTrue(len(result['entries']) >= 4) self.assertTrue(len(result['entries']) >= 4)
def test_nhl_videocenter(self): def test_nhl_videocenter(self):
@ -122,8 +126,8 @@ class TestPlaylists(unittest.TestCase):
ie = NHLVideocenterIE(dl) ie = NHLVideocenterIE(dl)
result = ie.extract('http://video.canucks.nhl.com/videocenter/console?catid=999') result = ie.extract('http://video.canucks.nhl.com/videocenter/console?catid=999')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['id'], u'999') self.assertEqual(result['id'], '999')
self.assertEqual(result['title'], u'Highlights') self.assertEqual(result['title'], 'Highlights')
self.assertEqual(len(result['entries']), 12) self.assertEqual(len(result['entries']), 12)
def test_bambuser_channel(self): def test_bambuser_channel(self):
@ -131,7 +135,7 @@ class TestPlaylists(unittest.TestCase):
ie = BambuserChannelIE(dl) ie = BambuserChannelIE(dl)
result = ie.extract('http://bambuser.com/channel/pixelversity') result = ie.extract('http://bambuser.com/channel/pixelversity')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['title'], u'pixelversity') self.assertEqual(result['title'], 'pixelversity')
self.assertTrue(len(result['entries']) >= 60) self.assertTrue(len(result['entries']) >= 60)
def test_bandcamp_album(self): def test_bandcamp_album(self):
@ -139,7 +143,7 @@ class TestPlaylists(unittest.TestCase):
ie = BandcampAlbumIE(dl) ie = BandcampAlbumIE(dl)
result = ie.extract('http://mpallante.bandcamp.com/album/nightmare-night-ep') result = ie.extract('http://mpallante.bandcamp.com/album/nightmare-night-ep')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['title'], u'Nightmare Night EP') self.assertEqual(result['title'], 'Nightmare Night EP')
self.assertTrue(len(result['entries']) >= 4) self.assertTrue(len(result['entries']) >= 4)
def test_smotri_community(self): def test_smotri_community(self):
@ -147,8 +151,8 @@ class TestPlaylists(unittest.TestCase):
ie = SmotriCommunityIE(dl) ie = SmotriCommunityIE(dl)
result = ie.extract('http://smotri.com/community/video/kommuna') result = ie.extract('http://smotri.com/community/video/kommuna')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['id'], u'kommuna') self.assertEqual(result['id'], 'kommuna')
self.assertEqual(result['title'], u'КПРФ') self.assertEqual(result['title'], 'КПРФ')
self.assertTrue(len(result['entries']) >= 4) self.assertTrue(len(result['entries']) >= 4)
def test_smotri_user(self): def test_smotri_user(self):
@ -156,17 +160,17 @@ class TestPlaylists(unittest.TestCase):
ie = SmotriUserIE(dl) ie = SmotriUserIE(dl)
result = ie.extract('http://smotri.com/user/inspector') result = ie.extract('http://smotri.com/user/inspector')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['id'], u'inspector') self.assertEqual(result['id'], 'inspector')
self.assertEqual(result['title'], u'Inspector') self.assertEqual(result['title'], 'Inspector')
self.assertTrue(len(result['entries']) >= 9) self.assertTrue(len(result['entries']) >= 9)
def test_AcademicEarthCourse(self): def test_AcademicEarthCourse(self):
dl = FakeYDL() dl = FakeYDL()
ie = AcademicEarthCourseIE(dl) ie = AcademicEarthCourseIE(dl)
result = ie.extract(u'http://academicearth.org/courses/building-dynamic-websites/') result = ie.extract('http://academicearth.org/courses/building-dynamic-websites/')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['id'], u'building-dynamic-websites') self.assertEqual(result['id'], 'building-dynamic-websites')
self.assertEqual(result['title'], u'Building Dynamic Websites') self.assertEqual(result['title'], 'Building Dynamic Websites')
self.assertEqual(result['description'], u"Today's websites are increasingly dynamic. Pages are no longer static HTML files but instead generated by scripts and database calls. User interfaces are more seamless, with technologies like Ajax replacing traditional page reloads. This course teaches students how to build dynamic websites with Ajax and with Linux, Apache, MySQL, and PHP (LAMP), one of today's most popular frameworks. Students learn how to set up domain names with DNS, how to structure pages with XHTML and CSS, how to program in JavaScript and PHP, how to configure Apache and MySQL, how to design and query databases with SQL, how to use Ajax with both XML and JSON, and how to build mashups. The course explores issues of security, scalability, and cross-browser support and also discusses enterprise-level deployments of websites, including third-party hosting, virtualization, colocation in data centers, firewalling, and load-balancing.") self.assertEqual(result['description'], u"Today's websites are increasingly dynamic. Pages are no longer static HTML files but instead generated by scripts and database calls. User interfaces are more seamless, with technologies like Ajax replacing traditional page reloads. This course teaches students how to build dynamic websites with Ajax and with Linux, Apache, MySQL, and PHP (LAMP), one of today's most popular frameworks. Students learn how to set up domain names with DNS, how to structure pages with XHTML and CSS, how to program in JavaScript and PHP, how to configure Apache and MySQL, how to design and query databases with SQL, how to use Ajax with both XML and JSON, and how to build mashups. The course explores issues of security, scalability, and cross-browser support and also discusses enterprise-level deployments of websites, including third-party hosting, virtualization, colocation in data centers, firewalling, and load-balancing.")
self.assertEqual(len(result['entries']), 10) self.assertEqual(len(result['entries']), 10)
@ -175,8 +179,8 @@ class TestPlaylists(unittest.TestCase):
ie = IviCompilationIE(dl) ie = IviCompilationIE(dl)
result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel') result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['id'], u'dezhurnyi_angel') self.assertEqual(result['id'], 'dezhurnyi_angel')
self.assertEqual(result['title'], u'Дежурный ангел (2010 - 2012)') self.assertEqual(result['title'], 'Дежурный ангел (2010 - 2012)')
self.assertTrue(len(result['entries']) >= 36) self.assertTrue(len(result['entries']) >= 36)
def test_ivi_compilation_season(self): def test_ivi_compilation_season(self):
@ -184,10 +188,38 @@ class TestPlaylists(unittest.TestCase):
ie = IviCompilationIE(dl) ie = IviCompilationIE(dl)
result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel/season2') result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel/season2')
self.assertIsPlaylist(result) self.assertIsPlaylist(result)
self.assertEqual(result['id'], u'dezhurnyi_angel/season2') self.assertEqual(result['id'], 'dezhurnyi_angel/season2')
self.assertEqual(result['title'], u'Дежурный ангел (2010 - 2012) 2 сезон') self.assertEqual(result['title'], 'Дежурный ангел (2010 - 2012) 2 сезон')
self.assertTrue(len(result['entries']) >= 20) self.assertTrue(len(result['entries']) >= 20)
def test_imdb_list(self):
dl = FakeYDL()
ie = ImdbListIE(dl)
result = ie.extract('http://www.imdb.com/list/sMjedvGDd8U')
self.assertIsPlaylist(result)
self.assertEqual(result['id'], 'sMjedvGDd8U')
self.assertEqual(result['title'], 'Animated and Family Films')
self.assertTrue(len(result['entries']) >= 48)
def test_khanacademy_topic(self):
dl = FakeYDL()
ie = KhanAcademyIE(dl)
result = ie.extract('https://www.khanacademy.org/math/applied-math/cryptography')
self.assertIsPlaylist(result)
self.assertEqual(result['id'], 'cryptography')
self.assertEqual(result['title'], 'Journey into cryptography')
self.assertEqual(result['description'], 'How have humans protected their secret messages through history? What has changed today?')
self.assertTrue(len(result['entries']) >= 3)
def test_EveryonesMixtape(self):
dl = FakeYDL()
ie = EveryonesMixtapeIE(dl)
result = ie.extract('http://everyonesmixtape.com/#/mix/m7m0jJAbMQi')
self.assertIsPlaylist(result)
self.assertEqual(result['id'], 'm7m0jJAbMQi')
self.assertEqual(result['title'], 'Driving')
self.assertEqual(len(result['entries']), 24)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

4
test/test_subtitles.py
View File

@ -167,13 +167,13 @@ class TestTedSubtitles(BaseTestSubtitles):
def test_subtitles(self): def test_subtitles(self):
self.DL.params['writesubtitles'] = True self.DL.params['writesubtitles'] = True
subtitles = self.getSubtitles() subtitles = self.getSubtitles()
self.assertEqual(md5(subtitles['en']), '2154f31ff9b9f89a0aa671537559c21d') self.assertEqual(md5(subtitles['en']), '4262c1665ff928a2dada178f62cb8d14')
def test_subtitles_lang(self): def test_subtitles_lang(self):
self.DL.params['writesubtitles'] = True self.DL.params['writesubtitles'] = True
self.DL.params['subtitleslangs'] = ['fr'] self.DL.params['subtitleslangs'] = ['fr']
subtitles = self.getSubtitles() subtitles = self.getSubtitles()
self.assertEqual(md5(subtitles['fr']), '7616cbc6df20ec2c1204083c83871cf6') self.assertEqual(md5(subtitles['fr']), '66a63f7f42c97a50f8c0e90bc7797bb5')
def test_allsubtitles(self): def test_allsubtitles(self):
self.DL.params['writesubtitles'] = True self.DL.params['writesubtitles'] = True

47
test/test_unicode_literals.py Normal file
View File

@ -0,0 +1,47 @@
from __future__ import unicode_literals
import io
import os
import re
import unittest
rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
IGNORED_FILES = [
'setup.py', # http://bugs.python.org/issue13943
]
class TestUnicodeLiterals(unittest.TestCase):
def test_all_files(self):
print('Skipping this test (not yet fully implemented)')
return
for dirpath, _, filenames in os.walk(rootDir):
for basename in filenames:
if not basename.endswith('.py'):
continue
if basename in IGNORED_FILES:
continue
fn = os.path.join(dirpath, basename)
with io.open(fn, encoding='utf-8') as inf:
code = inf.read()
if "'" not in code and '"' not in code:
continue
imps = 'from __future__ import unicode_literals'
self.assertTrue(
imps in code,
' %s missing in %s' % (imps, fn))
m = re.search(r'(?<=\s)u[\'"](?!\)|,|$)', code)
if m is not None:
self.assertTrue(
m is None,
'u present in %s, around %s' % (
fn, code[m.start() - 10:m.end() + 10]))
if __name__ == '__main__':
unittest.main()

6
test/test_youtube_signature.py
View File

@ -27,12 +27,6 @@ _TESTS = [
85, 85,
u'3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@', u'3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@',
), ),
(
u'https://s.ytimg.com/yts/swfbin/watch_as3-vflg5GhxU.swf',
u'swf',
82,
u':/.-,+*)=\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBAzyxw>utsrqponmlkjihgfedcba987654321'
),
] ]

313
youtube_dl/YoutubeDL.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import absolute_import from __future__ import absolute_import, unicode_literals
import collections import collections
import errno import errno
@ -51,9 +51,11 @@ from .utils import (
write_json_file, write_json_file,
write_string, write_string,
YoutubeDLHandler, YoutubeDLHandler,
prepend_extension,
) )
from .extractor import get_info_extractor, gen_extractors from .extractor import get_info_extractor, gen_extractors
from .downloader import get_suitable_downloader from .downloader import get_suitable_downloader
from .postprocessor import FFmpegMergerPP
from .version import __version__ from .version import __version__
@ -148,11 +150,16 @@ class YoutubeDL(object):
socket_timeout: Time to wait for unresponsive hosts, in seconds socket_timeout: Time to wait for unresponsive hosts, in seconds
bidi_workaround: Work around buggy terminals without bidirectional text bidi_workaround: Work around buggy terminals without bidirectional text
support, using fridibi support, using fridibi
debug_printtraffic:Print out sent and received HTTP traffic
The following parameters are not used by YoutubeDL itself, they are used by The following parameters are not used by YoutubeDL itself, they are used by
the FileDownloader: the FileDownloader:
nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test, nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
noresizebuffer, retries, continuedl, noprogress, consoletitle noresizebuffer, retries, continuedl, noprogress, consoletitle
The following options are used by the post processors:
prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
otherwise prefer avconv.
""" """
params = None params = None
@ -164,6 +171,8 @@ class YoutubeDL(object):
def __init__(self, params=None): def __init__(self, params=None):
"""Create a FileDownloader object with the given options.""" """Create a FileDownloader object with the given options."""
if params is None:
params = {}
self._ies = [] self._ies = []
self._ies_instances = {} self._ies_instances = {}
self._pps = [] self._pps = []
@ -172,7 +181,7 @@ class YoutubeDL(object):
self._num_downloads = 0 self._num_downloads = 0
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
self._err_file = sys.stderr self._err_file = sys.stderr
self.params = {} if params is None else params self.params = params
if params.get('bidi_workaround', False): if params.get('bidi_workaround', False):
try: try:
@ -197,7 +206,7 @@ class YoutubeDL(object):
self._output_channel = os.fdopen(master, 'rb') self._output_channel = os.fdopen(master, 'rb')
except OSError as ose: except OSError as ose:
if ose.errno == 2: if ose.errno == 2:
self.report_warning(u'Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.') self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
else: else:
raise raise
@ -206,13 +215,13 @@ class YoutubeDL(object):
and not params['restrictfilenames']): and not params['restrictfilenames']):
# On Python 3, the Unicode filesystem API will throw errors (#1474) # On Python 3, the Unicode filesystem API will throw errors (#1474)
self.report_warning( self.report_warning(
u'Assuming --restrict-filenames since file system encoding ' 'Assuming --restrict-filenames since file system encoding '
u'cannot encode all charactes. ' 'cannot encode all charactes. '
u'Set the LC_ALL environment variable to fix this.') 'Set the LC_ALL environment variable to fix this.')
self.params['restrictfilenames'] = True self.params['restrictfilenames'] = True
if '%(stitle)s' in self.params.get('outtmpl', ''): if '%(stitle)s' in self.params.get('outtmpl', ''):
self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
self._setup_opener() self._setup_opener()
@ -255,13 +264,13 @@ class YoutubeDL(object):
return message return message
assert hasattr(self, '_output_process') assert hasattr(self, '_output_process')
assert type(message) == type(u'') assert type(message) == type('')
line_count = message.count(u'\n') + 1 line_count = message.count('\n') + 1
self._output_process.stdin.write((message + u'\n').encode('utf-8')) self._output_process.stdin.write((message + '\n').encode('utf-8'))
self._output_process.stdin.flush() self._output_process.stdin.flush()
res = u''.join(self._output_channel.readline().decode('utf-8') res = ''.join(self._output_channel.readline().decode('utf-8')
for _ in range(line_count)) for _ in range(line_count))
return res[:-len(u'\n')] return res[:-len('\n')]
def to_screen(self, message, skip_eol=False): def to_screen(self, message, skip_eol=False):
"""Print message to stdout if not in quiet mode.""" """Print message to stdout if not in quiet mode."""
@ -273,19 +282,19 @@ class YoutubeDL(object):
self.params['logger'].debug(message) self.params['logger'].debug(message)
elif not check_quiet or not self.params.get('quiet', False): elif not check_quiet or not self.params.get('quiet', False):
message = self._bidi_workaround(message) message = self._bidi_workaround(message)
terminator = [u'\n', u''][skip_eol] terminator = ['\n', ''][skip_eol]
output = message + terminator output = message + terminator
write_string(output, self._screen_file) write_string(output, self._screen_file)
def to_stderr(self, message): def to_stderr(self, message):
"""Print message to stderr.""" """Print message to stderr."""
assert type(message) == type(u'') assert type(message) == type('')
if self.params.get('logger'): if self.params.get('logger'):
self.params['logger'].error(message) self.params['logger'].error(message)
else: else:
message = self._bidi_workaround(message) message = self._bidi_workaround(message)
output = message + u'\n' output = message + '\n'
write_string(output, self._err_file) write_string(output, self._err_file)
def to_console_title(self, message): def to_console_title(self, message):
@ -296,21 +305,21 @@ class YoutubeDL(object):
# already of type unicode() # already of type unicode()
ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
elif 'TERM' in os.environ: elif 'TERM' in os.environ:
write_string(u'\033]0;%s\007' % message, self._screen_file) write_string('\033]0;%s\007' % message, self._screen_file)
def save_console_title(self): def save_console_title(self):
if not self.params.get('consoletitle', False): if not self.params.get('consoletitle', False):
return return
if 'TERM' in os.environ: if 'TERM' in os.environ:
# Save the title on stack # Save the title on stack
write_string(u'\033[22;0t', self._screen_file) write_string('\033[22;0t', self._screen_file)
def restore_console_title(self): def restore_console_title(self):
if not self.params.get('consoletitle', False): if not self.params.get('consoletitle', False):
return return
if 'TERM' in os.environ: if 'TERM' in os.environ:
# Restore the title from stack # Restore the title from stack
write_string(u'\033[23;0t', self._screen_file) write_string('\033[23;0t', self._screen_file)
def __enter__(self): def __enter__(self):
self.save_console_title() self.save_console_title()
@ -336,13 +345,13 @@ class YoutubeDL(object):
if self.params.get('verbose'): if self.params.get('verbose'):
if tb is None: if tb is None:
if sys.exc_info()[0]: # if .trouble has been called from an except block if sys.exc_info()[0]: # if .trouble has been called from an except block
tb = u'' tb = ''
if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]: if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info)) tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
tb += compat_str(traceback.format_exc()) tb += compat_str(traceback.format_exc())
else: else:
tb_data = traceback.format_list(traceback.extract_stack()) tb_data = traceback.format_list(traceback.extract_stack())
tb = u''.join(tb_data) tb = ''.join(tb_data)
self.to_stderr(tb) self.to_stderr(tb)
if not self.params.get('ignoreerrors', False): if not self.params.get('ignoreerrors', False):
if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]: if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
@ -358,10 +367,10 @@ class YoutubeDL(object):
If stderr is a tty file the 'WARNING:' will be colored If stderr is a tty file the 'WARNING:' will be colored
''' '''
if self._err_file.isatty() and os.name != 'nt': if self._err_file.isatty() and os.name != 'nt':
_msg_header = u'\033[0;33mWARNING:\033[0m' _msg_header = '\033[0;33mWARNING:\033[0m'
else: else:
_msg_header = u'WARNING:' _msg_header = 'WARNING:'
warning_message = u'%s %s' % (_msg_header, message) warning_message = '%s %s' % (_msg_header, message)
self.to_stderr(warning_message) self.to_stderr(warning_message)
def report_error(self, message, tb=None): def report_error(self, message, tb=None):
@ -370,18 +379,18 @@ class YoutubeDL(object):
in red if stderr is a tty file. in red if stderr is a tty file.
''' '''
if self._err_file.isatty() and os.name != 'nt': if self._err_file.isatty() and os.name != 'nt':
_msg_header = u'\033[0;31mERROR:\033[0m' _msg_header = '\033[0;31mERROR:\033[0m'
else: else:
_msg_header = u'ERROR:' _msg_header = 'ERROR:'
error_message = u'%s %s' % (_msg_header, message) error_message = '%s %s' % (_msg_header, message)
self.trouble(error_message, tb) self.trouble(error_message, tb)
def report_file_already_downloaded(self, file_name): def report_file_already_downloaded(self, file_name):
"""Report file has already been fully downloaded.""" """Report file has already been fully downloaded."""
try: try:
self.to_screen(u'[download] %s has already been downloaded' % file_name) self.to_screen('[download] %s has already been downloaded' % file_name)
except UnicodeEncodeError: except UnicodeEncodeError:
self.to_screen(u'[download] The file has already been downloaded') self.to_screen('[download] The file has already been downloaded')
def increment_downloads(self): def increment_downloads(self):
"""Increment the ordinal that assigns a number to each file.""" """Increment the ordinal that assigns a number to each file."""
@ -396,61 +405,61 @@ class YoutubeDL(object):
autonumber_size = self.params.get('autonumber_size') autonumber_size = self.params.get('autonumber_size')
if autonumber_size is None: if autonumber_size is None:
autonumber_size = 5 autonumber_size = 5
autonumber_templ = u'%0' + str(autonumber_size) + u'd' autonumber_templ = '%0' + str(autonumber_size) + 'd'
template_dict['autonumber'] = autonumber_templ % self._num_downloads template_dict['autonumber'] = autonumber_templ % self._num_downloads
if template_dict.get('playlist_index') is not None: if template_dict.get('playlist_index') is not None:
template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index'] template_dict['playlist_index'] = '%05d' % template_dict['playlist_index']
sanitize = lambda k, v: sanitize_filename( sanitize = lambda k, v: sanitize_filename(
compat_str(v), compat_str(v),
restricted=self.params.get('restrictfilenames'), restricted=self.params.get('restrictfilenames'),
is_id=(k == u'id')) is_id=(k == 'id'))
template_dict = dict((k, sanitize(k, v)) template_dict = dict((k, sanitize(k, v))
for k, v in template_dict.items() for k, v in template_dict.items()
if v is not None) if v is not None)
template_dict = collections.defaultdict(lambda: u'NA', template_dict) template_dict = collections.defaultdict(lambda: 'NA', template_dict)
tmpl = os.path.expanduser(self.params['outtmpl']) tmpl = os.path.expanduser(self.params['outtmpl'])
filename = tmpl % template_dict filename = tmpl % template_dict
return filename return filename
except ValueError as err: except ValueError as err:
self.report_error(u'Error in output template: ' + str(err) + u' (encoding: ' + repr(preferredencoding()) + ')') self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
return None return None
def _match_entry(self, info_dict): def _match_entry(self, info_dict):
""" Returns None iff the file should be downloaded """ """ Returns None iff the file should be downloaded """
video_title = info_dict.get('title', info_dict.get('id', u'video')) video_title = info_dict.get('title', info_dict.get('id', 'video'))
if 'title' in info_dict: if 'title' in info_dict:
# This can happen when we're just evaluating the playlist # This can happen when we're just evaluating the playlist
title = info_dict['title'] title = info_dict['title']
matchtitle = self.params.get('matchtitle', False) matchtitle = self.params.get('matchtitle', False)
if matchtitle: if matchtitle:
if not re.search(matchtitle, title, re.IGNORECASE): if not re.search(matchtitle, title, re.IGNORECASE):
return u'"' + title + '" title did not match pattern "' + matchtitle + '"' return '"' + title + '" title did not match pattern "' + matchtitle + '"'
rejecttitle = self.params.get('rejecttitle', False) rejecttitle = self.params.get('rejecttitle', False)
if rejecttitle: if rejecttitle:
if re.search(rejecttitle, title, re.IGNORECASE): if re.search(rejecttitle, title, re.IGNORECASE):
return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
date = info_dict.get('upload_date', None) date = info_dict.get('upload_date', None)
if date is not None: if date is not None:
dateRange = self.params.get('daterange', DateRange()) dateRange = self.params.get('daterange', DateRange())
if date not in dateRange: if date not in dateRange:
return u'%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange) return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
view_count = info_dict.get('view_count', None) view_count = info_dict.get('view_count', None)
if view_count is not None: if view_count is not None:
min_views = self.params.get('min_views') min_views = self.params.get('min_views')
if min_views is not None and view_count < min_views: if min_views is not None and view_count < min_views:
return u'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views) return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
max_views = self.params.get('max_views') max_views = self.params.get('max_views')
if max_views is not None and view_count > max_views: if max_views is not None and view_count > max_views:
return u'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views) return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
age_limit = self.params.get('age_limit') age_limit = self.params.get('age_limit')
if age_limit is not None: if age_limit is not None:
if age_limit < info_dict.get('age_limit', 0): if age_limit < info_dict.get('age_limit', 0):
return u'Skipping "' + title + '" because it is age restricted' return 'Skipping "' + title + '" because it is age restricted'
if self.in_download_archive(info_dict): if self.in_download_archive(info_dict):
return u'%s has already been recorded in archive' % video_title return '%s has already been recorded in archive' % video_title
return None return None
@staticmethod @staticmethod
@ -477,8 +486,8 @@ class YoutubeDL(object):
continue continue
if not ie.working(): if not ie.working():
self.report_warning(u'The program functionality for this site has been marked as broken, ' self.report_warning('The program functionality for this site has been marked as broken, '
u'and will probably not work.') 'and will probably not work.')
try: try:
ie_result = ie.extract(url) ie_result = ie.extract(url)
@ -511,7 +520,7 @@ class YoutubeDL(object):
else: else:
raise raise
else: else:
self.report_error(u'no suitable InfoExtractor: %s' % url) self.report_error('no suitable InfoExtractor: %s' % url)
def process_ie_result(self, ie_result, download=True, extra_info={}): def process_ie_result(self, ie_result, download=True, extra_info={}):
""" """
@ -562,7 +571,7 @@ class YoutubeDL(object):
elif result_type == 'playlist': elif result_type == 'playlist':
# We process each entry in the playlist # We process each entry in the playlist
playlist = ie_result.get('title', None) or ie_result.get('id', None) playlist = ie_result.get('title', None) or ie_result.get('id', None)
self.to_screen(u'[download] Downloading playlist: %s' % playlist) self.to_screen('[download] Downloading playlist: %s' % playlist)
playlist_results = [] playlist_results = []
@ -577,11 +586,11 @@ class YoutubeDL(object):
n_entries = len(entries) n_entries = len(entries)
self.to_screen( self.to_screen(
u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" % "[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
(ie_result['extractor'], playlist, n_all_entries, n_entries)) (ie_result['extractor'], playlist, n_all_entries, n_entries))
for i, entry in enumerate(entries, 1): for i, entry in enumerate(entries, 1):
self.to_screen(u'[download] Downloading video #%s of %s' % (i, n_entries)) self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries))
extra = { extra = {
'playlist': playlist, 'playlist': playlist,
'playlist_index': i + playliststart, 'playlist_index': i + playliststart,
@ -593,7 +602,7 @@ class YoutubeDL(object):
reason = self._match_entry(entry) reason = self._match_entry(entry)
if reason is not None: if reason is not None:
self.to_screen(u'[download] ' + reason) self.to_screen('[download] ' + reason)
continue continue
entry_result = self.process_ie_result(entry, entry_result = self.process_ie_result(entry,
@ -626,7 +635,7 @@ class YoutubeDL(object):
elif format_spec == 'worst': elif format_spec == 'worst':
return available_formats[0] return available_formats[0]
else: else:
extensions = [u'mp4', u'flv', u'webm', u'3gp'] extensions = ['mp4', 'flv', 'webm', '3gp']
if format_spec in extensions: if format_spec in extensions:
filter_f = lambda f: f['ext'] == format_spec filter_f = lambda f: f['ext'] == format_spec
else: else:
@ -645,7 +654,7 @@ class YoutubeDL(object):
info_dict['playlist_index'] = None info_dict['playlist_index'] = None
# This extractors handle format selection themselves # This extractors handle format selection themselves
if info_dict['extractor'] in [u'Youku']: if info_dict['extractor'] in ['Youku']:
if download: if download:
self.process_info(info_dict) self.process_info(info_dict)
return info_dict return info_dict
@ -662,10 +671,10 @@ class YoutubeDL(object):
if format.get('format_id') is None: if format.get('format_id') is None:
format['format_id'] = compat_str(i) format['format_id'] = compat_str(i)
if format.get('format') is None: if format.get('format') is None:
format['format'] = u'{id} - {res}{note}'.format( format['format'] = '{id} - {res}{note}'.format(
id=format['format_id'], id=format['format_id'],
res=self.format_resolution(format), res=self.format_resolution(format),
note=u' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '', note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
) )
# Automatically determine file extension if missing # Automatically determine file extension if missing
if 'ext' not in format: if 'ext' not in format:
@ -697,21 +706,35 @@ class YoutubeDL(object):
if req_format in ('-1', 'all'): if req_format in ('-1', 'all'):
formats_to_download = formats formats_to_download = formats
else: else:
# We can accept formats requestd in the format: 34/5/best, we pick # We can accept formats requested in the format: 34/5/best, we pick
# the first that is available, starting from left # the first that is available, starting from left
req_formats = req_format.split('/') req_formats = req_format.split('/')
for rf in req_formats: for rf in req_formats:
selected_format = self.select_format(rf, formats) if re.match(r'.+?\+.+?', rf) is not None:
# Two formats have been requested like '137+139'
format_1, format_2 = rf.split('+')
formats_info = (self.select_format(format_1, formats),
self.select_format(format_2, formats))
if all(formats_info):
selected_format = {
'requested_formats': formats_info,
'format': rf,
'ext': formats_info[0]['ext'],
}
else:
selected_format = None
else:
selected_format = self.select_format(rf, formats)
if selected_format is not None: if selected_format is not None:
formats_to_download = [selected_format] formats_to_download = [selected_format]
break break
if not formats_to_download: if not formats_to_download:
raise ExtractorError(u'requested format not available', raise ExtractorError('requested format not available',
expected=True) expected=True)
if download: if download:
if len(formats_to_download) > 1: if len(formats_to_download) > 1:
self.to_screen(u'[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download))) self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
for format in formats_to_download: for format in formats_to_download:
new_info = dict(info_dict) new_info = dict(info_dict)
new_info.update(format) new_info.update(format)
@ -729,7 +752,7 @@ class YoutubeDL(object):
info_dict['fulltitle'] = info_dict['title'] info_dict['fulltitle'] = info_dict['title']
if len(info_dict['title']) > 200: if len(info_dict['title']) > 200:
info_dict['title'] = info_dict['title'][:197] + u'...' info_dict['title'] = info_dict['title'][:197] + '...'
# Keep for backwards compatibility # Keep for backwards compatibility
info_dict['stitle'] = info_dict['title'] info_dict['stitle'] = info_dict['title']
@ -739,7 +762,7 @@ class YoutubeDL(object):
reason = self._match_entry(info_dict) reason = self._match_entry(info_dict)
if reason is not None: if reason is not None:
self.to_screen(u'[download] ' + reason) self.to_screen('[download] ' + reason)
return return
max_downloads = self.params.get('max_downloads') max_downloads = self.params.get('max_downloads')
@ -756,7 +779,7 @@ class YoutubeDL(object):
self.to_stdout(info_dict['id']) self.to_stdout(info_dict['id'])
if self.params.get('forceurl', False): if self.params.get('forceurl', False):
# For RTMP URLs, also include the playpath # For RTMP URLs, also include the playpath
self.to_stdout(info_dict['url'] + info_dict.get('play_path', u'')) self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None: if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
self.to_stdout(info_dict['thumbnail']) self.to_stdout(info_dict['thumbnail'])
if self.params.get('forcedescription', False) and info_dict.get('description') is not None: if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
@ -783,37 +806,37 @@ class YoutubeDL(object):
if dn != '' and not os.path.exists(dn): if dn != '' and not os.path.exists(dn):
os.makedirs(dn) os.makedirs(dn)
except (OSError, IOError) as err: except (OSError, IOError) as err:
self.report_error(u'unable to create directory ' + compat_str(err)) self.report_error('unable to create directory ' + compat_str(err))
return return
if self.params.get('writedescription', False): if self.params.get('writedescription', False):
descfn = filename + u'.description' descfn = filename + '.description'
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)): if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
self.to_screen(u'[info] Video description is already present') self.to_screen('[info] Video description is already present')
else: else:
try: try:
self.to_screen(u'[info] Writing video description to: ' + descfn) self.to_screen('[info] Writing video description to: ' + descfn)
with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile: with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
descfile.write(info_dict['description']) descfile.write(info_dict['description'])
except (KeyError, TypeError): except (KeyError, TypeError):
self.report_warning(u'There\'s no description to write.') self.report_warning('There\'s no description to write.')
except (OSError, IOError): except (OSError, IOError):
self.report_error(u'Cannot write description file ' + descfn) self.report_error('Cannot write description file ' + descfn)
return return
if self.params.get('writeannotations', False): if self.params.get('writeannotations', False):
annofn = filename + u'.annotations.xml' annofn = filename + '.annotations.xml'
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)): if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
self.to_screen(u'[info] Video annotations are already present') self.to_screen('[info] Video annotations are already present')
else: else:
try: try:
self.to_screen(u'[info] Writing video annotations to: ' + annofn) self.to_screen('[info] Writing video annotations to: ' + annofn)
with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile: with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
annofile.write(info_dict['annotations']) annofile.write(info_dict['annotations'])
except (KeyError, TypeError): except (KeyError, TypeError):
self.report_warning(u'There are no annotations to write.') self.report_warning('There are no annotations to write.')
except (OSError, IOError): except (OSError, IOError):
self.report_error(u'Cannot write annotations file: ' + annofn) self.report_error('Cannot write annotations file: ' + annofn)
return return
subtitles_are_requested = any([self.params.get('writesubtitles', False), subtitles_are_requested = any([self.params.get('writesubtitles', False),
@ -831,45 +854,45 @@ class YoutubeDL(object):
try: try:
sub_filename = subtitles_filename(filename, sub_lang, sub_format) sub_filename = subtitles_filename(filename, sub_lang, sub_format)
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)): if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
self.to_screen(u'[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format)) self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
else: else:
self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename) self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
subfile.write(sub) subfile.write(sub)
except (OSError, IOError): except (OSError, IOError):
self.report_error(u'Cannot write subtitles file ' + descfn) self.report_error('Cannot write subtitles file ' + descfn)
return return
if self.params.get('writeinfojson', False): if self.params.get('writeinfojson', False):
infofn = os.path.splitext(filename)[0] + u'.info.json' infofn = os.path.splitext(filename)[0] + '.info.json'
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
self.to_screen(u'[info] Video description metadata is already present') self.to_screen('[info] Video description metadata is already present')
else: else:
self.to_screen(u'[info] Writing video description metadata as JSON to: ' + infofn) self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
try: try:
write_json_file(info_dict, encodeFilename(infofn)) write_json_file(info_dict, encodeFilename(infofn))
except (OSError, IOError): except (OSError, IOError):
self.report_error(u'Cannot write metadata to JSON file ' + infofn) self.report_error('Cannot write metadata to JSON file ' + infofn)
return return
if self.params.get('writethumbnail', False): if self.params.get('writethumbnail', False):
if info_dict.get('thumbnail') is not None: if info_dict.get('thumbnail') is not None:
thumb_format = determine_ext(info_dict['thumbnail'], u'jpg') thumb_format = determine_ext(info_dict['thumbnail'], 'jpg')
thumb_filename = os.path.splitext(filename)[0] + u'.' + thumb_format thumb_filename = os.path.splitext(filename)[0] + '.' + thumb_format
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)): if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
self.to_screen(u'[%s] %s: Thumbnail is already present' % self.to_screen('[%s] %s: Thumbnail is already present' %
(info_dict['extractor'], info_dict['id'])) (info_dict['extractor'], info_dict['id']))
else: else:
self.to_screen(u'[%s] %s: Downloading thumbnail ...' % self.to_screen('[%s] %s: Downloading thumbnail ...' %
(info_dict['extractor'], info_dict['id'])) (info_dict['extractor'], info_dict['id']))
try: try:
uf = compat_urllib_request.urlopen(info_dict['thumbnail']) uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
with open(thumb_filename, 'wb') as thumbf: with open(thumb_filename, 'wb') as thumbf:
shutil.copyfileobj(uf, thumbf) shutil.copyfileobj(uf, thumbf)
self.to_screen(u'[%s] %s: Writing thumbnail to: %s' % self.to_screen('[%s] %s: Writing thumbnail to: %s' %
(info_dict['extractor'], info_dict['id'], thumb_filename)) (info_dict['extractor'], info_dict['id'], thumb_filename))
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self.report_warning(u'Unable to download thumbnail "%s": %s' % self.report_warning('Unable to download thumbnail "%s": %s' %
(info_dict['thumbnail'], compat_str(err))) (info_dict['thumbnail'], compat_str(err)))
if not self.params.get('skip_download', False): if not self.params.get('skip_download', False):
@ -877,24 +900,49 @@ class YoutubeDL(object):
success = True success = True
else: else:
try: try:
fd = get_suitable_downloader(info_dict)(self, self.params) def dl(name, info):
for ph in self._progress_hooks: fd = get_suitable_downloader(info)(self, self.params)
fd.add_progress_hook(ph) for ph in self._progress_hooks:
success = fd.download(filename, info_dict) fd.add_progress_hook(ph)
return fd.download(name, info)
if info_dict.get('requested_formats') is not None:
downloaded = []
success = True
merger = FFmpegMergerPP(self)
if not merger._get_executable():
postprocessors = []
self.report_warning('You have requested multiple '
'formats but ffmpeg or avconv are not installed.'
' The formats won\'t be merged')
else:
postprocessors = [merger]
for f in info_dict['requested_formats']:
new_info = dict(info_dict)
new_info.update(f)
fname = self.prepare_filename(new_info)
fname = prepend_extension(fname, 'f%s' % f['format_id'])
downloaded.append(fname)
partial_success = dl(fname, new_info)
success = success and partial_success
info_dict['__postprocessors'] = postprocessors
info_dict['__files_to_merge'] = downloaded
else:
# Just a single file
success = dl(filename, info_dict)
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self.report_error(u'unable to download video data: %s' % str(err)) self.report_error('unable to download video data: %s' % str(err))
return return
except (OSError, IOError) as err: except (OSError, IOError) as err:
raise UnavailableVideoError(err) raise UnavailableVideoError(err)
except (ContentTooShortError, ) as err: except (ContentTooShortError, ) as err:
self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
return return
if success: if success:
try: try:
self.post_process(filename, info_dict) self.post_process(filename, info_dict)
except (PostProcessingError) as err: except (PostProcessingError) as err:
self.report_error(u'postprocessing: %s' % str(err)) self.report_error('postprocessing: %s' % str(err))
return return
self.record_download_archive(info_dict) self.record_download_archive(info_dict)
@ -911,9 +959,9 @@ class YoutubeDL(object):
#It also downloads the videos #It also downloads the videos
self.extract_info(url) self.extract_info(url)
except UnavailableVideoError: except UnavailableVideoError:
self.report_error(u'unable to download video') self.report_error('unable to download video')
except MaxDownloadsReached: except MaxDownloadsReached:
self.to_screen(u'[info] Maximum number of downloaded files reached.') self.to_screen('[info] Maximum number of downloaded files reached.')
raise raise
return self._download_retcode return self._download_retcode
@ -926,7 +974,7 @@ class YoutubeDL(object):
except DownloadError: except DownloadError:
webpage_url = info.get('webpage_url') webpage_url = info.get('webpage_url')
if webpage_url is not None: if webpage_url is not None:
self.report_warning(u'The info failed to download, trying with "%s"' % webpage_url) self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
return self.download([webpage_url]) return self.download([webpage_url])
else: else:
raise raise
@ -937,7 +985,11 @@ class YoutubeDL(object):
info = dict(ie_info) info = dict(ie_info)
info['filepath'] = filename info['filepath'] = filename
keep_video = None keep_video = None
for pp in self._pps: pps_chain = []
if ie_info.get('__postprocessors') is not None:
pps_chain.extend(ie_info['__postprocessors'])
pps_chain.extend(self._pps)
for pp in pps_chain:
try: try:
keep_video_wish, new_info = pp.run(info) keep_video_wish, new_info = pp.run(info)
if keep_video_wish is not None: if keep_video_wish is not None:
@ -950,10 +1002,10 @@ class YoutubeDL(object):
self.report_error(e.msg) self.report_error(e.msg)
if keep_video is False and not self.params.get('keepvideo', False): if keep_video is False and not self.params.get('keepvideo', False):
try: try:
self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename) self.to_screen('Deleting original file %s (pass -k to keep)' % filename)
os.remove(encodeFilename(filename)) os.remove(encodeFilename(filename))
except (IOError, OSError): except (IOError, OSError):
self.report_warning(u'Unable to remove downloaded video file') self.report_warning('Unable to remove downloaded video file')
def _make_archive_id(self, info_dict): def _make_archive_id(self, info_dict):
# Future-proof against any change in case # Future-proof against any change in case
@ -964,7 +1016,7 @@ class YoutubeDL(object):
extractor = info_dict.get('ie_key') # key in a playlist extractor = info_dict.get('ie_key') # key in a playlist
if extractor is None: if extractor is None:
return None # Incomplete video information return None # Incomplete video information
return extractor.lower() + u' ' + info_dict['id'] return extractor.lower() + ' ' + info_dict['id']
def in_download_archive(self, info_dict): def in_download_archive(self, info_dict):
fn = self.params.get('download_archive') fn = self.params.get('download_archive')
@ -992,7 +1044,7 @@ class YoutubeDL(object):
vid_id = self._make_archive_id(info_dict) vid_id = self._make_archive_id(info_dict)
assert vid_id assert vid_id
with locked_file(fn, 'a', encoding='utf-8') as archive_file: with locked_file(fn, 'a', encoding='utf-8') as archive_file:
archive_file.write(vid_id + u'\n') archive_file.write(vid_id + '\n')
@staticmethod @staticmethod
def format_resolution(format, default='unknown'): def format_resolution(format, default='unknown'):
@ -1002,49 +1054,51 @@ class YoutubeDL(object):
return format['resolution'] return format['resolution']
if format.get('height') is not None: if format.get('height') is not None:
if format.get('width') is not None: if format.get('width') is not None:
res = u'%sx%s' % (format['width'], format['height']) res = '%sx%s' % (format['width'], format['height'])
else: else:
res = u'%sp' % format['height'] res = '%sp' % format['height']
elif format.get('width') is not None: elif format.get('width') is not None:
res = u'?x%d' % format['width'] res = '?x%d' % format['width']
else: else:
res = default res = default
return res return res
def list_formats(self, info_dict): def list_formats(self, info_dict):
def format_note(fdict): def format_note(fdict):
res = u'' res = ''
if f.get('ext') in ['f4f', 'f4m']: if fdict.get('ext') in ['f4f', 'f4m']:
res += u'(unsupported) ' res += '(unsupported) '
if fdict.get('format_note') is not None: if fdict.get('format_note') is not None:
res += fdict['format_note'] + u' ' res += fdict['format_note'] + ' '
if fdict.get('tbr') is not None: if fdict.get('tbr') is not None:
res += u'%4dk ' % fdict['tbr'] res += '%4dk ' % fdict['tbr']
if (fdict.get('vcodec') is not None and if (fdict.get('vcodec') is not None and
fdict.get('vcodec') != 'none'): fdict.get('vcodec') != 'none'):
res += u'%-5s@' % fdict['vcodec'] res += '%-5s' % fdict['vcodec']
if fdict.get('vbr') is not None:
res += '@'
elif fdict.get('vbr') is not None and fdict.get('abr') is not None: elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
res += u'video@' res += 'video@'
if fdict.get('vbr') is not None: if fdict.get('vbr') is not None:
res += u'%4dk' % fdict['vbr'] res += '%4dk' % fdict['vbr']
if fdict.get('acodec') is not None: if fdict.get('acodec') is not None:
if res: if res:
res += u', ' res += ', '
res += u'%-5s' % fdict['acodec'] res += '%-5s' % fdict['acodec']
elif fdict.get('abr') is not None: elif fdict.get('abr') is not None:
if res: if res:
res += u', ' res += ', '
res += 'audio' res += 'audio'
if fdict.get('abr') is not None: if fdict.get('abr') is not None:
res += u'@%3dk' % fdict['abr'] res += '@%3dk' % fdict['abr']
if fdict.get('filesize') is not None: if fdict.get('filesize') is not None:
if res: if res:
res += u', ' res += ', '
res += format_bytes(fdict['filesize']) res += format_bytes(fdict['filesize'])
return res return res
def line(format, idlen=20): def line(format, idlen=20):
return ((u'%-' + compat_str(idlen + 1) + u's%-10s%-12s%s') % ( return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
format['format_id'], format['format_id'],
format['ext'], format['ext'],
self.format_resolution(format), self.format_resolution(format),
@ -1052,7 +1106,7 @@ class YoutubeDL(object):
)) ))
formats = info_dict.get('formats', [info_dict]) formats = info_dict.get('formats', [info_dict])
idlen = max(len(u'format code'), idlen = max(len('format code'),
max(len(f['format_id']) for f in formats)) max(len(f['format_id']) for f in formats))
formats_s = [line(f, idlen) for f in formats] formats_s = [line(f, idlen) for f in formats]
if len(formats) > 1: if len(formats) > 1:
@ -1060,10 +1114,10 @@ class YoutubeDL(object):
formats_s[-1] += (' ' if format_note(formats[-1]) else '') + '(best)' formats_s[-1] += (' ' if format_note(formats[-1]) else '') + '(best)'
header_line = line({ header_line = line({
'format_id': u'format code', 'ext': u'extension', 'format_id': 'format code', 'ext': 'extension',
'resolution': u'resolution', 'format_note': u'note'}, idlen=idlen) 'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
self.to_screen(u'[info] Available formats for %s:\n%s\n%s' % self.to_screen('[info] Available formats for %s:\n%s\n%s' %
(info_dict['id'], header_line, u"\n".join(formats_s))) (info_dict['id'], header_line, '\n'.join(formats_s)))
def urlopen(self, req): def urlopen(self, req):
""" Start an HTTP download """ """ Start an HTTP download """
@ -1072,7 +1126,7 @@ class YoutubeDL(object):
def print_debug_header(self): def print_debug_header(self):
if not self.params.get('verbose'): if not self.params.get('verbose'):
return return
write_string(u'[debug] youtube-dl version ' + __version__ + u'\n') write_string('[debug] youtube-dl version ' + __version__ + '\n')
try: try:
sp = subprocess.Popen( sp = subprocess.Popen(
['git', 'rev-parse', '--short', 'HEAD'], ['git', 'rev-parse', '--short', 'HEAD'],
@ -1081,20 +1135,20 @@ class YoutubeDL(object):
out, err = sp.communicate() out, err = sp.communicate()
out = out.decode().strip() out = out.decode().strip()
if re.match('[0-9a-f]+', out): if re.match('[0-9a-f]+', out):
write_string(u'[debug] Git HEAD: ' + out + u'\n') write_string('[debug] Git HEAD: ' + out + '\n')
except: except:
try: try:
sys.exc_clear() sys.exc_clear()
except: except:
pass pass
write_string(u'[debug] Python version %s - %s' % write_string('[debug] Python version %s - %s' %
(platform.python_version(), platform_name()) + u'\n') (platform.python_version(), platform_name()) + '\n')
proxy_map = {} proxy_map = {}
for handler in self._opener.handlers: for handler in self._opener.handlers:
if hasattr(handler, 'proxies'): if hasattr(handler, 'proxies'):
proxy_map.update(handler.proxies) proxy_map.update(handler.proxies)
write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n') write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
def _setup_opener(self): def _setup_opener(self):
timeout_val = self.params.get('socket_timeout') timeout_val = self.params.get('socket_timeout')
@ -1124,10 +1178,13 @@ class YoutubeDL(object):
if 'http' in proxies and 'https' not in proxies: if 'http' in proxies and 'https' not in proxies:
proxies['https'] = proxies['http'] proxies['https'] = proxies['http']
proxy_handler = compat_urllib_request.ProxyHandler(proxies) proxy_handler = compat_urllib_request.ProxyHandler(proxies)
debuglevel = 1 if self.params.get('debug_printtraffic') else 0
https_handler = make_HTTPS_handler( https_handler = make_HTTPS_handler(
self.params.get('nocheckcertificate', False)) self.params.get('nocheckcertificate', False), debuglevel=debuglevel)
ydlh = YoutubeDLHandler(debuglevel=debuglevel)
opener = compat_urllib_request.build_opener( opener = compat_urllib_request.build_opener(
https_handler, proxy_handler, cookie_processor, YoutubeDLHandler()) https_handler, proxy_handler, cookie_processor, ydlh)
# Delete the default user-agent header, which would otherwise apply in # Delete the default user-agent header, which would otherwise apply in
# cases where our custom HTTP handler doesn't come into play # cases where our custom HTTP handler doesn't come into play
# (See https://github.com/rg3/youtube-dl/issues/1309 for details) # (See https://github.com/rg3/youtube-dl/issues/1309 for details)

57
youtube_dl/__init__.py
View File

@ -38,12 +38,15 @@ __authors__ = (
'Takuya Tsuchida', 'Takuya Tsuchida',
'Sergey M.', 'Sergey M.',
'Michael Orlitzky', 'Michael Orlitzky',
'Chris Gahan',
'Saimadhav Heblikar',
) )
__license__ = 'Public Domain' __license__ = 'Public Domain'
import codecs import codecs
import getpass import getpass
import locale
import optparse import optparse
import os import os
import random import random
@ -73,11 +76,12 @@ from .FileDownloader import (
from .extractor import gen_extractors from .extractor import gen_extractors
from .version import __version__ from .version import __version__
from .YoutubeDL import YoutubeDL from .YoutubeDL import YoutubeDL
from .PostProcessor import ( from .postprocessor import (
FFmpegMetadataPP, FFmpegMetadataPP,
FFmpegVideoConvertor, FFmpegVideoConvertor,
FFmpegExtractAudioPP, FFmpegExtractAudioPP,
FFmpegEmbedSubtitlePP, FFmpegEmbedSubtitlePP,
XAttrMetadataPP,
) )
@ -185,13 +189,13 @@ def parseOpts(overrideArguments=None):
general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.') general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
general.add_option( general.add_option(
'--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR', '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
help='Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl .') help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
general.add_option( general.add_option(
'--no-cache-dir', action='store_const', const=None, dest='cachedir', '--no-cache-dir', action='store_const', const=None, dest='cachedir',
help='Disable filesystem caching') help='Disable filesystem caching')
general.add_option( general.add_option(
'--socket-timeout', dest='socket_timeout', '--socket-timeout', dest='socket_timeout',
type=float, default=None, help=optparse.SUPPRESS_HELP) type=float, default=None, help=u'Time to wait before giving up, in seconds')
general.add_option( general.add_option(
'--bidi-workaround', dest='bidi_workaround', action='store_true', '--bidi-workaround', dest='bidi_workaround', action='store_true',
help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH') help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
@ -213,8 +217,12 @@ def parseOpts(overrideArguments=None):
selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None) selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None) selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None) selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None) selection.add_option(
selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None) '--datebefore', metavar='DATE', dest='datebefore', default=None,
help='download only videos uploaded on or before this date (i.e. inclusive)')
selection.add_option(
'--dateafter', metavar='DATE', dest='dateafter', default=None,
help='download only videos uploaded on or after this date (i.e. inclusive)')
selection.add_option( selection.add_option(
'--min-views', metavar='COUNT', dest='min_views', '--min-views', metavar='COUNT', dest='min_views',
default=None, type=int, default=None, type=int,
@ -239,7 +247,7 @@ def parseOpts(overrideArguments=None):
authentication.add_option('-n', '--netrc', authentication.add_option('-n', '--netrc',
action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
authentication.add_option('--video-password', authentication.add_option('--video-password',
dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)') dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
video_format.add_option('-f', '--format', video_format.add_option('-f', '--format',
@ -252,7 +260,7 @@ def parseOpts(overrideArguments=None):
video_format.add_option('--max-quality', video_format.add_option('--max-quality',
action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
video_format.add_option('-F', '--list-formats', video_format.add_option('-F', '--list-formats',
action='store_true', dest='listformats', help='list all available formats (currently youtube only)') action='store_true', dest='listformats', help='list all available formats')
subtitles.add_option('--write-sub', '--write-srt', subtitles.add_option('--write-sub', '--write-srt',
action='store_true', dest='writesubtitles', action='store_true', dest='writesubtitles',
@ -326,14 +334,16 @@ def parseOpts(overrideArguments=None):
action='store_true', dest='verbose', help='print various debugging information', default=False) action='store_true', dest='verbose', help='print various debugging information', default=False)
verbosity.add_option('--dump-intermediate-pages', verbosity.add_option('--dump-intermediate-pages',
action='store_true', dest='dump_intermediate_pages', default=False, action='store_true', dest='dump_intermediate_pages', default=False,
help='print downloaded pages to debug problems(very verbose)') help='print downloaded pages to debug problems (very verbose)')
verbosity.add_option('--write-pages', verbosity.add_option('--write-pages',
action='store_true', dest='write_pages', default=False, action='store_true', dest='write_pages', default=False,
help='Write downloaded intermediary pages to files in the current directory to debug problems') help='Write downloaded intermediary pages to files in the current directory to debug problems')
verbosity.add_option('--youtube-print-sig-code', verbosity.add_option('--youtube-print-sig-code',
action='store_true', dest='youtube_print_sig_code', default=False, action='store_true', dest='youtube_print_sig_code', default=False,
help=optparse.SUPPRESS_HELP) help=optparse.SUPPRESS_HELP)
verbosity.add_option('--print-traffic',
dest='debug_printtraffic', action='store_true', default=False,
help=optparse.SUPPRESS_HELP)
filesystem.add_option('-t', '--title', filesystem.add_option('-t', '--title',
action='store_true', dest='usetitle', help='use title in file name (default)', default=False) action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
@ -350,11 +360,11 @@ def parseOpts(overrideArguments=None):
'%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, ' '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
'%(autonumber)s to get an automatically incremented number, ' '%(autonumber)s to get an automatically incremented number, '
'%(ext)s for the filename extension, ' '%(ext)s for the filename extension, '
'%(format)s for the format description (like "22 - 1280x720" or "HD"),' '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
'%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"),' '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
'%(upload_date)s for the upload date (YYYYMMDD), ' '%(upload_date)s for the upload date (YYYYMMDD), '
'%(extractor)s for the provider (youtube, metacafe, etc), ' '%(extractor)s for the provider (youtube, metacafe, etc), '
'%(id)s for the video id , %(playlist)s for the playlist the video is in, ' '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
'%(playlist_index)s for the position in the playlist and %% for a literal percent. ' '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
'Use - to output to stdout. Can also be used to download to a different directory, ' 'Use - to output to stdout. Can also be used to download to a different directory, '
'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')) 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
@ -368,7 +378,7 @@ def parseOpts(overrideArguments=None):
dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
filesystem.add_option('--load-info', filesystem.add_option('--load-info',
dest='load_info_filename', metavar='FILE', dest='load_info_filename', metavar='FILE',
help='json file containing the video information (created with the "--write-json" option') help='json file containing the video information (created with the "--write-json" option)')
filesystem.add_option('-w', '--no-overwrites', filesystem.add_option('-w', '--no-overwrites',
action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
filesystem.add_option('-c', '--continue', filesystem.add_option('-c', '--continue',
@ -412,7 +422,13 @@ def parseOpts(overrideArguments=None):
postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False, postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
help='embed subtitles in the video (only for mp4 videos)') help='embed subtitles in the video (only for mp4 videos)')
postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False, postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
help='add metadata to the files') help='write metadata to the video file')
postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
help='Prefer avconv over ffmpeg for running the postprocessors (default)')
postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
help='Prefer ffmpeg over avconv for running the postprocessors')
parser.add_option_group(general) parser.add_option_group(general)
@ -473,6 +489,8 @@ def parseOpts(overrideArguments=None):
write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n') write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n') write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n') write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
write_string(u'[debug] Encodings: locale %r, fs %r, out %r, pref: %r\n' %
(locale.getpreferredencoding(), sys.getfilesystemencoding(), sys.stdout.encoding, preferredencoding()))
return parser, opts, args return parser, opts, args
@ -517,6 +535,8 @@ def _real_main(argv=None):
sys.exit(u'ERROR: batch file could not be read') sys.exit(u'ERROR: batch file could not be read')
all_urls = batchurls + args all_urls = batchurls + args
all_urls = [url.strip() for url in all_urls] all_urls = [url.strip() for url in all_urls]
_enc = preferredencoding()
all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
extractors = gen_extractors() extractors = gen_extractors()
@ -546,7 +566,7 @@ def _real_main(argv=None):
if opts.usenetrc and (opts.username is not None or opts.password is not None): if opts.usenetrc and (opts.username is not None or opts.password is not None):
parser.error(u'using .netrc conflicts with giving username/password') parser.error(u'using .netrc conflicts with giving username/password')
if opts.password is not None and opts.username is None: if opts.password is not None and opts.username is None:
parser.error(u' account username missing\n') parser.error(u'account username missing\n')
if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid): if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
parser.error(u'using output template conflicts with using title, video ID or auto number') parser.error(u'using output template conflicts with using title, video ID or auto number')
if opts.usetitle and opts.useid: if opts.usetitle and opts.useid:
@ -620,6 +640,7 @@ def _real_main(argv=None):
u' template'.format(outtmpl)) u' template'.format(outtmpl))
any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
ydl_opts = { ydl_opts = {
'usenetrc': opts.usenetrc, 'usenetrc': opts.usenetrc,
@ -687,12 +708,14 @@ def _real_main(argv=None):
'cachedir': opts.cachedir, 'cachedir': opts.cachedir,
'youtube_print_sig_code': opts.youtube_print_sig_code, 'youtube_print_sig_code': opts.youtube_print_sig_code,
'age_limit': opts.age_limit, 'age_limit': opts.age_limit,
'download_archive': opts.download_archive, 'download_archive': download_archive_fn,
'cookiefile': opts.cookiefile, 'cookiefile': opts.cookiefile,
'nocheckcertificate': opts.no_check_certificate, 'nocheckcertificate': opts.no_check_certificate,
'proxy': opts.proxy, 'proxy': opts.proxy,
'socket_timeout': opts.socket_timeout, 'socket_timeout': opts.socket_timeout,
'bidi_workaround': opts.bidi_workaround, 'bidi_workaround': opts.bidi_workaround,
'debug_printtraffic': opts.debug_printtraffic,
'prefer_ffmpeg': opts.prefer_ffmpeg,
} }
with YoutubeDL(ydl_opts) as ydl: with YoutubeDL(ydl_opts) as ydl:
@ -709,6 +732,8 @@ def _real_main(argv=None):
ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo)) ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
if opts.embedsubtitles: if opts.embedsubtitles:
ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat)) ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
if opts.xattrs:
ydl.add_post_processor(XAttrMetadataPP())
# Update version # Update version
if opts.update_self: if opts.update_self:

2
youtube_dl/downloader/hls.py
View File

@ -29,7 +29,7 @@ class HlsFD(FileDownloader):
retval = subprocess.call(cmd) retval = subprocess.call(cmd)
if retval == 0: if retval == 0:
fsize = os.path.getsize(encodeFilename(tmpfilename)) fsize = os.path.getsize(encodeFilename(tmpfilename))
self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) self.to_screen(u'\r[%s] %s bytes' % (cmd[0], fsize))
self.try_rename(tmpfilename, filename) self.try_rename(tmpfilename, filename)
self._hook_progress({ self._hook_progress({
'downloaded_bytes': fsize, 'downloaded_bytes': fsize,

2
youtube_dl/downloader/http.py
View File

@ -133,7 +133,7 @@ class HttpFD(FileDownloader):
return False return False
try: try:
stream.write(data_block) stream.write(data_block)
except (IOError, OSError): except (IOError, OSError) as err:
self.to_stderr(u"\n") self.to_stderr(u"\n")
self.report_error(u'unable to write data: %s' % str(err)) self.report_error(u'unable to write data: %s' % str(err))
return False return False

21
youtube_dl/extractor/__init__.py
View File

@ -28,6 +28,7 @@ from .channel9 import Channel9IE
from .cinemassacre import CinemassacreIE from .cinemassacre import CinemassacreIE
from .clipfish import ClipfishIE from .clipfish import ClipfishIE
from .clipsyndicate import ClipsyndicateIE from .clipsyndicate import ClipsyndicateIE
from .cmt import CMTIE
from .cnn import CNNIE from .cnn import CNNIE
from .collegehumor import CollegeHumorIE from .collegehumor import CollegeHumorIE
from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
@ -51,6 +52,7 @@ from .ehow import EHowIE
from .eighttracks import EightTracksIE from .eighttracks import EightTracksIE
from .eitb import EitbIE from .eitb import EitbIE
from .escapist import EscapistIE from .escapist import EscapistIE
from .everyonesmixtape import EveryonesMixtapeIE
from .exfm import ExfmIE from .exfm import ExfmIE
from .extremetube import ExtremeTubeIE from .extremetube import ExtremeTubeIE
from .facebook import FacebookIE from .facebook import FacebookIE
@ -60,11 +62,13 @@ from .fktv import (
FKTVPosteckeIE, FKTVPosteckeIE,
) )
from .flickr import FlickrIE from .flickr import FlickrIE
from .franceinter import FranceInterIE
from .francetv import ( from .francetv import (
PluzzIE, PluzzIE,
FranceTvInfoIE, FranceTvInfoIE,
FranceTVIE, FranceTVIE,
GenerationQuoiIE GenerationQuoiIE,
CultureboxIE,
) )
from .freesound import FreesoundIE from .freesound import FreesoundIE
from .funnyordie import FunnyOrDieIE from .funnyordie import FunnyOrDieIE
@ -79,7 +83,10 @@ from .hotnewhiphop import HotNewHipHopIE
from .howcast import HowcastIE from .howcast import HowcastIE
from .hypem import HypemIE from .hypem import HypemIE
from .ign import IGNIE, OneUPIE from .ign import IGNIE, OneUPIE
from .imdb import ImdbIE from .imdb import (
ImdbIE,
ImdbListIE
)
from .ina import InaIE from .ina import InaIE
from .infoq import InfoQIE from .infoq import InfoQIE
from .instagram import InstagramIE from .instagram import InstagramIE
@ -91,17 +98,25 @@ from .ivi import (
from .jeuxvideo import JeuxVideoIE from .jeuxvideo import JeuxVideoIE
from .jukebox import JukeboxIE from .jukebox import JukeboxIE
from .justintv import JustinTVIE from .justintv import JustinTVIE
from .jpopsukitv import JpopsukiIE
from .kankan import KankanIE from .kankan import KankanIE
from .keezmovies import KeezMoviesIE from .keezmovies import KeezMoviesIE
from .khanacademy import KhanAcademyIE
from .kickstarter import KickStarterIE from .kickstarter import KickStarterIE
from .keek import KeekIE from .keek import KeekIE
from .liveleak import LiveLeakIE from .liveleak import LiveLeakIE
from .livestream import LivestreamIE, LivestreamOriginalIE from .livestream import LivestreamIE, LivestreamOriginalIE
from .lynda import (
LyndaIE,
LyndaCourseIE
)
from .macgamestore import MacGameStoreIE
from .mdr import MDRIE from .mdr import MDRIE
from .metacafe import MetacafeIE from .metacafe import MetacafeIE
from .metacritic import MetacriticIE from .metacritic import MetacriticIE
from .mit import TechTVMITIE, MITIE from .mit import TechTVMITIE, MITIE
from .mixcloud import MixcloudIE from .mixcloud import MixcloudIE
from .mpora import MporaIE
from .mofosex import MofosexIE from .mofosex import MofosexIE
from .mtv import MTVIE from .mtv import MTVIE
from .muzu import MuzuTVIE from .muzu import MuzuTVIE
@ -116,6 +131,7 @@ from .newgrounds import NewgroundsIE
from .nhl import NHLIE, NHLVideocenterIE from .nhl import NHLIE, NHLVideocenterIE
from .niconico import NiconicoIE from .niconico import NiconicoIE
from .ninegag import NineGagIE from .ninegag import NineGagIE
from .novamov import NovamovIE
from .nowvideo import NowVideoIE from .nowvideo import NowVideoIE
from .ooyala import OoyalaIE from .ooyala import OoyalaIE
from .orf import ORFIE from .orf import ORFIE
@ -189,6 +205,7 @@ from .vimeo import (
VimeoUserIE, VimeoUserIE,
VimeoAlbumIE, VimeoAlbumIE,
VimeoGroupsIE, VimeoGroupsIE,
VimeoReviewIE,
) )
from .vine import VineIE from .vine import VineIE
from .viki import VikiIE from .viki import VikiIE

3
youtube_dl/extractor/academicearth.py
View File

@ -1,3 +1,4 @@
from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
@ -5,7 +6,7 @@ from .common import InfoExtractor
class AcademicEarthCourseIE(InfoExtractor): class AcademicEarthCourseIE(InfoExtractor):
_VALID_URL = r'^https?://(?:www\.)?academicearth\.org/(?:courses|playlists)/(?P<id>[^?#/]+)' _VALID_URL = r'^https?://(?:www\.)?academicearth\.org/(?:courses|playlists)/(?P<id>[^?#/]+)'
IE_NAME = u'AcademicEarth:Course' IE_NAME = 'AcademicEarth:Course'
def _real_extract(self, url): def _real_extract(self, url):
m = re.match(self._VALID_URL, url) m = re.match(self._VALID_URL, url)

62
youtube_dl/extractor/appletrailers.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import re import re
import json import json
@ -11,46 +13,46 @@ from ..utils import (
class AppleTrailersIE(InfoExtractor): class AppleTrailersIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)' _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)'
_TEST = { _TEST = {
u"url": u"http://trailers.apple.com/trailers/wb/manofsteel/", "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
u"playlist": [ "playlist": [
{ {
u"file": u"manofsteel-trailer4.mov", "file": "manofsteel-trailer4.mov",
u"md5": u"d97a8e575432dbcb81b7c3acb741f8a8", "md5": "d97a8e575432dbcb81b7c3acb741f8a8",
u"info_dict": { "info_dict": {
u"duration": 111, "duration": 111,
u"title": u"Trailer 4", "title": "Trailer 4",
u"upload_date": u"20130523", "upload_date": "20130523",
u"uploader_id": u"wb", "uploader_id": "wb",
}, },
}, },
{ {
u"file": u"manofsteel-trailer3.mov", "file": "manofsteel-trailer3.mov",
u"md5": u"b8017b7131b721fb4e8d6f49e1df908c", "md5": "b8017b7131b721fb4e8d6f49e1df908c",
u"info_dict": { "info_dict": {
u"duration": 182, "duration": 182,
u"title": u"Trailer 3", "title": "Trailer 3",
u"upload_date": u"20130417", "upload_date": "20130417",
u"uploader_id": u"wb", "uploader_id": "wb",
}, },
}, },
{ {
u"file": u"manofsteel-trailer.mov", "file": "manofsteel-trailer.mov",
u"md5": u"d0f1e1150989b9924679b441f3404d48", "md5": "d0f1e1150989b9924679b441f3404d48",
u"info_dict": { "info_dict": {
u"duration": 148, "duration": 148,
u"title": u"Trailer", "title": "Trailer",
u"upload_date": u"20121212", "upload_date": "20121212",
u"uploader_id": u"wb", "uploader_id": "wb",
}, },
}, },
{ {
u"file": u"manofsteel-teaser.mov", "file": "manofsteel-teaser.mov",
u"md5": u"5fe08795b943eb2e757fa95cb6def1cb", "md5": "5fe08795b943eb2e757fa95cb6def1cb",
u"info_dict": { "info_dict": {
u"duration": 93, "duration": 93,
u"title": u"Teaser", "title": "Teaser",
u"upload_date": u"20120721", "upload_date": "20120721",
u"uploader_id": u"wb", "uploader_id": "wb",
}, },
} }
] ]

32
youtube_dl/extractor/archiveorg.py
View File

@ -1,9 +1,10 @@
from __future__ import unicode_literals
import json import json
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
determine_ext,
unified_strdate, unified_strdate,
) )
@ -13,23 +14,22 @@ class ArchiveOrgIE(InfoExtractor):
IE_DESC = 'archive.org videos' IE_DESC = 'archive.org videos'
_VALID_URL = r'(?:https?://)?(?:www\.)?archive\.org/details/(?P<id>[^?/]+)(?:[?].*)?$' _VALID_URL = r'(?:https?://)?(?:www\.)?archive\.org/details/(?P<id>[^?/]+)(?:[?].*)?$'
_TEST = { _TEST = {
u"url": u"http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect", "url": "http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect",
u'file': u'XD300-23_68HighlightsAResearchCntAugHumanIntellect.ogv', 'file': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect.ogv',
u'md5': u'8af1d4cf447933ed3c7f4871162602db', 'md5': '8af1d4cf447933ed3c7f4871162602db',
u'info_dict': { 'info_dict': {
u"title": u"1968 Demo - FJCC Conference Presentation Reel #1", "title": "1968 Demo - FJCC Conference Presentation Reel #1",
u"description": u"Reel 1 of 3: Also known as the \"Mother of All Demos\", Doug Engelbart's presentation at the Fall Joint Computer Conference in San Francisco, December 9, 1968 titled \"A Research Center for Augmenting Human Intellect.\" For this presentation, Doug and his team astonished the audience by not only relating their research, but demonstrating it live. This was the debut of the mouse, interactive computing, hypermedia, computer supported software engineering, video teleconferencing, etc. See also <a href=\"http://dougengelbart.org/firsts/dougs-1968-demo.html\" rel=\"nofollow\">Doug's 1968 Demo page</a> for more background, highlights, links, and the detailed paper published in this conference proceedings. Filmed on 3 reels: Reel 1 | <a href=\"http://www.archive.org/details/XD300-24_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 2</a> | <a href=\"http://www.archive.org/details/XD300-25_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 3</a>", "description": "Reel 1 of 3: Also known as the \"Mother of All Demos\", Doug Engelbart's presentation at the Fall Joint Computer Conference in San Francisco, December 9, 1968 titled \"A Research Center for Augmenting Human Intellect.\" For this presentation, Doug and his team astonished the audience by not only relating their research, but demonstrating it live. This was the debut of the mouse, interactive computing, hypermedia, computer supported software engineering, video teleconferencing, etc. See also <a href=\"http://dougengelbart.org/firsts/dougs-1968-demo.html\" rel=\"nofollow\">Doug's 1968 Demo page</a> for more background, highlights, links, and the detailed paper published in this conference proceedings. Filmed on 3 reels: Reel 1 | <a href=\"http://www.archive.org/details/XD300-24_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 2</a> | <a href=\"http://www.archive.org/details/XD300-25_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 3</a>",
u"upload_date": u"19681210", "upload_date": "19681210",
u"uploader": u"SRI International" "uploader": "SRI International"
} }
} }
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id') video_id = mobj.group('id')
json_url = url + (u'?' if u'?' in url else '&') + u'output=json' json_url = url + ('?' if '?' in url else '&') + 'output=json'
json_data = self._download_webpage(json_url, video_id) json_data = self._download_webpage(json_url, video_id)
data = json.loads(json_data) data = json.loads(json_data)
@ -38,16 +38,16 @@ class ArchiveOrgIE(InfoExtractor):
uploader = data['metadata']['creator'][0] uploader = data['metadata']['creator'][0]
upload_date = unified_strdate(data['metadata']['date'][0]) upload_date = unified_strdate(data['metadata']['date'][0])
formats = [{ formats = [
{
'format': fdata['format'], 'format': fdata['format'],
'url': 'http://' + data['server'] + data['dir'] + fn, 'url': 'http://' + data['server'] + data['dir'] + fn,
'file_size': int(fdata['size']), 'file_size': int(fdata['size']),
} }
for fn,fdata in data['files'].items() for fn, fdata in data['files'].items()
if 'Video' in fdata['format']] if 'Video' in fdata['format']]
formats.sort(key=lambda fdata: fdata['file_size'])
for f in formats: self._sort_formats(formats)
f['ext'] = determine_ext(f['url'])
return { return {
'_type': 'video', '_type': 'video',

46
youtube_dl/extractor/arte.py
View File

@ -1,4 +1,6 @@
# encoding: utf-8 # encoding: utf-8
from __future__ import unicode_literals
import re import re
import json import json
@ -22,7 +24,7 @@ class ArteTvIE(InfoExtractor):
_LIVEWEB_URL = r'(?:http://)?liveweb\.arte\.tv/(?P<lang>fr|de)/(?P<subpage>.+?)/(?P<name>.+)' _LIVEWEB_URL = r'(?:http://)?liveweb\.arte\.tv/(?P<lang>fr|de)/(?P<subpage>.+?)/(?P<name>.+)'
_LIVE_URL = r'index-[0-9]+\.html$' _LIVE_URL = r'index-[0-9]+\.html$'
IE_NAME = u'arte.tv' IE_NAME = 'arte.tv'
@classmethod @classmethod
def suitable(cls, url): def suitable(cls, url):
@ -37,7 +39,7 @@ class ArteTvIE(InfoExtractor):
# r'src="(.*?/videothek_js.*?\.js)', # r'src="(.*?/videothek_js.*?\.js)',
# 0, # 0,
# [ # [
# (1, 'url', u'Invalid URL: %s' % url) # (1, 'url', 'Invalid URL: %s' % url)
# ] # ]
# ) # )
# http_host = url.split('/')[2] # http_host = url.split('/')[2]
@ -49,12 +51,12 @@ class ArteTvIE(InfoExtractor):
# '(rtmp://.*?)\'', # '(rtmp://.*?)\'',
# re.DOTALL, # re.DOTALL,
# [ # [
# (1, 'path', u'could not extract video path: %s' % url), # (1, 'path', 'could not extract video path: %s' % url),
# (2, 'player', u'could not extract video player: %s' % url), # (2, 'player', 'could not extract video player: %s' % url),
# (3, 'url', u'could not extract video url: %s' % url) # (3, 'url', 'could not extract video url: %s' % url)
# ] # ]
# ) # )
# video_url = u'%s/%s' % (info.get('url'), info.get('path')) # video_url = '%s/%s' % (info.get('url'), info.get('path'))
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VIDEOS_URL, url) mobj = re.match(self._VIDEOS_URL, url)
@ -107,9 +109,9 @@ class ArteTvIE(InfoExtractor):
def _extract_liveweb(self, url, name, lang): def _extract_liveweb(self, url, name, lang):
"""Extract form http://liveweb.arte.tv/""" """Extract form http://liveweb.arte.tv/"""
webpage = self._download_webpage(url, name) webpage = self._download_webpage(url, name)
video_id = self._search_regex(r'eventId=(\d+?)("|&)', webpage, u'event id') video_id = self._search_regex(r'eventId=(\d+?)("|&)', webpage, 'event id')
config_doc = self._download_xml('http://download.liveweb.arte.tv/o21/liveweb/events/event-%s.xml' % video_id, config_doc = self._download_xml('http://download.liveweb.arte.tv/o21/liveweb/events/event-%s.xml' % video_id,
video_id, u'Downloading information') video_id, 'Downloading information')
event_doc = config_doc.find('event') event_doc = config_doc.find('event')
url_node = event_doc.find('video').find('urlHd') url_node = event_doc.find('video').find('urlHd')
if url_node is None: if url_node is None:
@ -124,7 +126,7 @@ class ArteTvIE(InfoExtractor):
class ArteTVPlus7IE(InfoExtractor): class ArteTVPlus7IE(InfoExtractor):
IE_NAME = u'arte.tv:+7' IE_NAME = 'arte.tv:+7'
_VALID_URL = r'https?://www\.arte.tv/guide/(?P<lang>fr|de)/(?:(?:sendungen|emissions)/)?(?P<id>.*?)/(?P<name>.*?)(\?.*)?' _VALID_URL = r'https?://www\.arte.tv/guide/(?P<lang>fr|de)/(?:(?:sendungen|emissions)/)?(?P<id>.*?)/(?P<name>.*?)(\?.*)?'
@classmethod @classmethod
@ -207,7 +209,7 @@ class ArteTVPlus7IE(InfoExtractor):
if bitrate is not None: if bitrate is not None:
quality += '-%d' % bitrate quality += '-%d' % bitrate
if format_info.get('versionCode') is not None: if format_info.get('versionCode') is not None:
format_id = u'%s-%s' % (quality, format_info['versionCode']) format_id = '%s-%s' % (quality, format_info['versionCode'])
else: else:
format_id = quality format_id = quality
info = { info = {
@ -216,7 +218,7 @@ class ArteTVPlus7IE(InfoExtractor):
'width': format_info.get('width'), 'width': format_info.get('width'),
'height': height, 'height': height,
} }
if format_info['mediaType'] == u'rtmp': if format_info['mediaType'] == 'rtmp':
info['url'] = format_info['streamer'] info['url'] = format_info['streamer']
info['play_path'] = 'mp4:' + format_info['url'] info['play_path'] = 'mp4:' + format_info['url']
info['ext'] = 'flv' info['ext'] = 'flv'
@ -231,27 +233,27 @@ class ArteTVPlus7IE(InfoExtractor):
# It also uses the arte_vp_url url from the webpage to extract the information # It also uses the arte_vp_url url from the webpage to extract the information
class ArteTVCreativeIE(ArteTVPlus7IE): class ArteTVCreativeIE(ArteTVPlus7IE):
IE_NAME = u'arte.tv:creative' IE_NAME = 'arte.tv:creative'
_VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de)/magazine?/(?P<id>.+)' _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de)/magazine?/(?P<id>.+)'
_TEST = { _TEST = {
u'url': u'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design', 'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
u'file': u'050489-002.mp4', 'file': '050489-002.mp4',
u'info_dict': { 'info_dict': {
u'title': u'Agentur Amateur / Agence Amateur #2 : Corporate Design', 'title': 'Agentur Amateur / Agence Amateur #2 : Corporate Design',
}, },
} }
class ArteTVFutureIE(ArteTVPlus7IE): class ArteTVFutureIE(ArteTVPlus7IE):
IE_NAME = u'arte.tv:future' IE_NAME = 'arte.tv:future'
_VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(thema|sujet)/.*?#article-anchor-(?P<id>\d+)' _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(thema|sujet)/.*?#article-anchor-(?P<id>\d+)'
_TEST = { _TEST = {
u'url': u'http://future.arte.tv/fr/sujet/info-sciences#article-anchor-7081', 'url': 'http://future.arte.tv/fr/sujet/info-sciences#article-anchor-7081',
u'file': u'050940-003.mp4', 'file': '050940-003.mp4',
u'info_dict': { 'info_dict': {
u'title': u'Les champignons au secours de la planète', 'title': 'Les champignons au secours de la planète',
}, },
} }
@ -263,7 +265,7 @@ class ArteTVFutureIE(ArteTVPlus7IE):
class ArteTVDDCIE(ArteTVPlus7IE): class ArteTVDDCIE(ArteTVPlus7IE):
IE_NAME = u'arte.tv:ddc' IE_NAME = 'arte.tv:ddc'
_VALID_URL = r'http?://ddc\.arte\.tv/(?P<lang>emission|folge)/(?P<id>.+)' _VALID_URL = r'http?://ddc\.arte\.tv/(?P<lang>emission|folge)/(?P<id>.+)'
def _real_extract(self, url): def _real_extract(self, url):

17
youtube_dl/extractor/auengine.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
@ -7,13 +9,14 @@ from ..utils import (
ExtractorError, ExtractorError,
) )
class AUEngineIE(InfoExtractor): class AUEngineIE(InfoExtractor):
_TEST = { _TEST = {
u'url': u'http://auengine.com/embed.php?file=lfvlytY6&w=650&h=370', 'url': 'http://auengine.com/embed.php?file=lfvlytY6&w=650&h=370',
u'file': u'lfvlytY6.mp4', 'file': 'lfvlytY6.mp4',
u'md5': u'48972bdbcf1a3a2f5533e62425b41d4f', 'md5': '48972bdbcf1a3a2f5533e62425b41d4f',
u'info_dict': { 'info_dict': {
u"title": u"[Commie]The Legend of the Legendary Heroes - 03 - Replication Eye (Alpha Stigma)[F9410F5A]" 'title': '[Commie]The Legend of the Legendary Heroes - 03 - Replication Eye (Alpha Stigma)[F9410F5A]'
} }
} }
_VALID_URL = r'(?:http://)?(?:www\.)?auengine\.com/embed\.php\?.*?file=([^&]+).*?' _VALID_URL = r'(?:http://)?(?:www\.)?auengine\.com/embed\.php\?.*?file=([^&]+).*?'
@ -23,7 +26,7 @@ class AUEngineIE(InfoExtractor):
video_id = mobj.group(1) video_id = mobj.group(1)
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
title = self._html_search_regex(r'<title>(?P<title>.+?)</title>', title = self._html_search_regex(r'<title>(?P<title>.+?)</title>',
webpage, u'title') webpage, 'title')
title = title.strip() title = title.strip()
links = re.findall(r'\s(?:file|url):\s*["\']([^\'"]+)["\']', webpage) links = re.findall(r'\s(?:file|url):\s*["\']([^\'"]+)["\']', webpage)
links = map(compat_urllib_parse.unquote, links) links = map(compat_urllib_parse.unquote, links)
@ -37,7 +40,7 @@ class AUEngineIE(InfoExtractor):
video_url = link video_url = link
if not video_url: if not video_url:
raise ExtractorError(u'Could not find video URL') raise ExtractorError(u'Could not find video URL')
ext = u'.' + determine_ext(video_url) ext = '.' + determine_ext(video_url)
if ext == title[-len(ext):]: if ext == title[-len(ext):]:
title = title[:-len(ext)] title = title[:-len(ext)]

30
youtube_dl/extractor/bambuser.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import re import re
import json import json
import itertools import itertools
@ -9,26 +11,26 @@ from ..utils import (
class BambuserIE(InfoExtractor): class BambuserIE(InfoExtractor):
IE_NAME = u'bambuser' IE_NAME = 'bambuser'
_VALID_URL = r'https?://bambuser\.com/v/(?P<id>\d+)' _VALID_URL = r'https?://bambuser\.com/v/(?P<id>\d+)'
_API_KEY = '005f64509e19a868399060af746a00aa' _API_KEY = '005f64509e19a868399060af746a00aa'
_TEST = { _TEST = {
u'url': u'http://bambuser.com/v/4050584', 'url': 'http://bambuser.com/v/4050584',
# MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388 # MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388
#u'md5': u'fba8f7693e48fd4e8641b3fd5539a641', #u'md5': 'fba8f7693e48fd4e8641b3fd5539a641',
u'info_dict': { 'info_dict': {
u'id': u'4050584', 'id': '4050584',
u'ext': u'flv', 'ext': 'flv',
u'title': u'Education engineering days - lightning talks', 'title': 'Education engineering days - lightning talks',
u'duration': 3741, 'duration': 3741,
u'uploader': u'pixelversity', 'uploader': 'pixelversity',
u'uploader_id': u'344706', 'uploader_id': '344706',
}, },
u'params': { 'params': {
# It doesn't respect the 'Range' header, it would download the whole video # It doesn't respect the 'Range' header, it would download the whole video
# caused the travis builds to fail: https://travis-ci.org/rg3/youtube-dl/jobs/14493845#L59 # caused the travis builds to fail: https://travis-ci.org/rg3/youtube-dl/jobs/14493845#L59
u'skip_download': True, 'skip_download': True,
}, },
} }
@ -53,7 +55,7 @@ class BambuserIE(InfoExtractor):
class BambuserChannelIE(InfoExtractor): class BambuserChannelIE(InfoExtractor):
IE_NAME = u'bambuser:channel' IE_NAME = 'bambuser:channel'
_VALID_URL = r'https?://bambuser\.com/channel/(?P<user>.*?)(?:/|#|\?|$)' _VALID_URL = r'https?://bambuser\.com/channel/(?P<user>.*?)(?:/|#|\?|$)'
# The maximum number we can get with each request # The maximum number we can get with each request
_STEP = 50 _STEP = 50
@ -72,7 +74,7 @@ class BambuserChannelIE(InfoExtractor):
# Without setting this header, we wouldn't get any result # Without setting this header, we wouldn't get any result
req.add_header('Referer', 'http://bambuser.com/channel/%s' % user) req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
info_json = self._download_webpage(req, user, info_json = self._download_webpage(req, user,
u'Downloading page %d' % i) 'Downloading page %d' % i)
results = json.loads(info_json)['result'] results = json.loads(info_json)['result']
if len(results) == 0: if len(results) == 0:
break break

115
youtube_dl/extractor/bandcamp.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import json import json
import re import re
@ -10,16 +12,16 @@ from ..utils import (
class BandcampIE(InfoExtractor): class BandcampIE(InfoExtractor):
IE_NAME = u'Bandcamp'
_VALID_URL = r'http://.*?\.bandcamp\.com/track/(?P<title>.*)' _VALID_URL = r'http://.*?\.bandcamp\.com/track/(?P<title>.*)'
_TESTS = [{ _TESTS = [{
u'url': u'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song', 'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
u'file': u'1812978515.mp3', 'file': '1812978515.mp3',
u'md5': u'cdeb30cdae1921719a3cbcab696ef53c', 'md5': 'c557841d5e50261777a6585648adf439',
u'info_dict': { 'info_dict': {
u"title": u"youtube-dl test song \"'/\\\u00e4\u21ad" "title": "youtube-dl \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad",
"duration": 10,
}, },
u'skip': u'There is a limit of 200 free downloads / month for the test song' '_skip': 'There is a limit of 200 free downloads / month for the test song'
}] }]
def _real_extract(self, url): def _real_extract(self, url):
@ -30,85 +32,98 @@ class BandcampIE(InfoExtractor):
m_download = re.search(r'freeDownloadPage: "(.*?)"', webpage) m_download = re.search(r'freeDownloadPage: "(.*?)"', webpage)
if m_download is None: if m_download is None:
m_trackinfo = re.search(r'trackinfo: (.+),\s*?\n', webpage) m_trackinfo = re.search(r'trackinfo: (.+),\s*?\n', webpage)
if m_trackinfo: if m_trackinfo:
json_code = m_trackinfo.group(1) json_code = m_trackinfo.group(1)
data = json.loads(json_code) data = json.loads(json_code)
d = data[0]
duration = int(round(d['duration']))
formats = []
for format_id, format_url in d['file'].items():
ext, _, abr_str = format_id.partition('-')
formats.append({
'format_id': format_id,
'url': format_url,
'ext': format_id.partition('-')[0],
'vcodec': 'none',
'acodec': format_id.partition('-')[0],
'abr': int(format_id.partition('-')[2]),
})
self._sort_formats(formats)
for d in data:
formats = [{
'format_id': 'format_id',
'url': format_url,
'ext': format_id.partition('-')[0]
} for format_id, format_url in sorted(d['file'].items())]
return { return {
'id': compat_str(d['id']), 'id': compat_str(d['id']),
'title': d['title'], 'title': d['title'],
'formats': formats, 'formats': formats,
'duration': duration,
} }
else: else:
raise ExtractorError(u'No free songs found') raise ExtractorError('No free songs found')
download_link = m_download.group(1) download_link = m_download.group(1)
id = re.search(r'var TralbumData = {(.*?)id: (?P<id>\d*?)$', video_id = re.search(
webpage, re.MULTILINE|re.DOTALL).group('id') r'var TralbumData = {(.*?)id: (?P<id>\d*?)$',
webpage, re.MULTILINE | re.DOTALL).group('id')
download_webpage = self._download_webpage(download_link, id, download_webpage = self._download_webpage(download_link, video_id,
'Downloading free downloads page') 'Downloading free downloads page')
# We get the dictionary of the track from some javascrip code # We get the dictionary of the track from some javascrip code
info = re.search(r'items: (.*?),$', info = re.search(r'items: (.*?),$',
download_webpage, re.MULTILINE).group(1) download_webpage, re.MULTILINE).group(1)
info = json.loads(info)[0] info = json.loads(info)[0]
# We pick mp3-320 for now, until format selection can be easily implemented. # We pick mp3-320 for now, until format selection can be easily implemented.
mp3_info = info[u'downloads'][u'mp3-320'] mp3_info = info['downloads']['mp3-320']
# If we try to use this url it says the link has expired # If we try to use this url it says the link has expired
initial_url = mp3_info[u'url'] initial_url = mp3_info['url']
re_url = r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$' re_url = r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$'
m_url = re.match(re_url, initial_url) m_url = re.match(re_url, initial_url)
#We build the url we will use to get the final track url #We build the url we will use to get the final track url
# This url is build in Bandcamp in the script download_bunde_*.js # This url is build in Bandcamp in the script download_bunde_*.js
request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), id, m_url.group('ts')) request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), video_id, m_url.group('ts'))
final_url_webpage = self._download_webpage(request_url, id, 'Requesting download url') final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url')
# If we could correctly generate the .rand field the url would be # If we could correctly generate the .rand field the url would be
#in the "download_url" key #in the "download_url" key
final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1) final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1)
track_info = {'id':id, return {
'title' : info[u'title'], 'id': video_id,
'ext' : 'mp3', 'title': info['title'],
'url' : final_url, 'ext': 'mp3',
'thumbnail' : info[u'thumb_url'], 'vcodec': 'none',
'uploader' : info[u'artist'] 'url': final_url,
} 'thumbnail': info.get('thumb_url'),
'uploader': info.get('artist'),
return [track_info] }
class BandcampAlbumIE(InfoExtractor): class BandcampAlbumIE(InfoExtractor):
IE_NAME = u'Bandcamp:album' IE_NAME = 'Bandcamp:album'
_VALID_URL = r'http://.*?\.bandcamp\.com/album/(?P<title>.*)' _VALID_URL = r'http://.*?\.bandcamp\.com/album/(?P<title>.*)'
_TEST = { _TEST = {
u'url': u'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1', 'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
u'playlist': [ 'playlist': [
{ {
u'file': u'1353101989.mp3', 'file': '1353101989.mp3',
u'md5': u'39bc1eded3476e927c724321ddf116cf', 'md5': '39bc1eded3476e927c724321ddf116cf',
u'info_dict': { 'info_dict': {
u'title': u'Intro', 'title': 'Intro',
} }
}, },
{ {
u'file': u'38097443.mp3', 'file': '38097443.mp3',
u'md5': u'1a2c32e2691474643e912cc6cd4bffaa', 'md5': '1a2c32e2691474643e912cc6cd4bffaa',
u'info_dict': { 'info_dict': {
u'title': u'Kero One - Keep It Alive (Blazo remix)', 'title': 'Kero One - Keep It Alive (Blazo remix)',
} }
}, },
], ],
u'params': { 'params': {
u'playlistend': 2 'playlistend': 2
}, },
u'skip': u'Bancamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test' 'skip': 'Bancamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test'
} }
def _real_extract(self, url): def _real_extract(self, url):
@ -117,11 +132,11 @@ class BandcampAlbumIE(InfoExtractor):
webpage = self._download_webpage(url, title) webpage = self._download_webpage(url, title)
tracks_paths = re.findall(r'<a href="(.*?)" itemprop="url">', webpage) tracks_paths = re.findall(r'<a href="(.*?)" itemprop="url">', webpage)
if not tracks_paths: if not tracks_paths:
raise ExtractorError(u'The page doesn\'t contain any track') raise ExtractorError('The page doesn\'t contain any tracks')
entries = [ entries = [
self.url_result(compat_urlparse.urljoin(url, t_path), ie=BandcampIE.ie_key()) self.url_result(compat_urlparse.urljoin(url, t_path), ie=BandcampIE.ie_key())
for t_path in tracks_paths] for t_path in tracks_paths]
title = self._search_regex(r'album_title : "(.*?)"', webpage, u'title') title = self._search_regex(r'album_title : "(.*?)"', webpage, 'title')
return { return {
'_type': 'playlist', '_type': 'playlist',
'title': title, 'title': title,

41
youtube_dl/extractor/blinkx.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import datetime import datetime
import json import json
import re import re
@ -10,19 +12,19 @@ from ..utils import (
class BlinkxIE(InfoExtractor): class BlinkxIE(InfoExtractor):
_VALID_URL = r'^(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)' _VALID_URL = r'^(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
_IE_NAME = u'blinkx' IE_NAME = 'blinkx'
_TEST = { _TEST = {
u'url': u'http://www.blinkx.com/ce/8aQUy7GVFYgFzpKhT0oqsilwOGFRVXk3R1ZGWWdGenBLaFQwb3FzaWx3OGFRVXk3R1ZGWWdGenB', 'url': 'http://www.blinkx.com/ce/8aQUy7GVFYgFzpKhT0oqsilwOGFRVXk3R1ZGWWdGenBLaFQwb3FzaWx3OGFRVXk3R1ZGWWdGenB',
u'file': u'8aQUy7GV.mp4', 'file': '8aQUy7GV.mp4',
u'md5': u'2e9a07364af40163a908edbf10bb2492', 'md5': '2e9a07364af40163a908edbf10bb2492',
u'info_dict': { 'info_dict': {
u"title": u"Police Car Rolls Away", "title": "Police Car Rolls Away",
u"uploader": u"stupidvideos.com", "uploader": "stupidvideos.com",
u"upload_date": u"20131215", "upload_date": "20131215",
u"description": u"A police car gently rolls away from a fight. Maybe it felt weird being around a confrontation and just had to get out of there!", "description": "A police car gently rolls away from a fight. Maybe it felt weird being around a confrontation and just had to get out of there!",
u"duration": 14.886, "duration": 14.886,
u"thumbnails": [{ "thumbnails": [{
"width": 100, "width": 100,
"height": 76, "height": 76,
"url": "http://cdn.blinkx.com/stream/b/41/StupidVideos/20131215/1873969261/1873969261_tn_0.jpg", "url": "http://cdn.blinkx.com/stream/b/41/StupidVideos/20131215/1873969261/1873969261_tn_0.jpg",
@ -30,17 +32,17 @@ class BlinkxIE(InfoExtractor):
}, },
} }
def _real_extract(self, url): def _real_extract(self, rl):
m = re.match(self._VALID_URL, url) m = re.match(self._VALID_URL, rl)
video_id = m.group('id') video_id = m.group('id')
display_id = video_id[:8] display_id = video_id[:8]
api_url = (u'https://apib4.blinkx.com/api.php?action=play_video&' + api_url = (u'https://apib4.blinkx.com/api.php?action=play_video&' +
u'video=%s' % video_id) 'video=%s' % video_id)
data_json = self._download_webpage(api_url, display_id) data_json = self._download_webpage(api_url, display_id)
data = json.loads(data_json)['api']['results'][0] data = json.loads(data_json)['api']['results'][0]
dt = datetime.datetime.fromtimestamp(data['pubdate_epoch']) dt = datetime.datetime.fromtimestamp(data['pubdate_epoch'])
upload_date = dt.strftime('%Y%m%d') pload_date = dt.strftime('%Y%m%d')
duration = None duration = None
thumbnails = [] thumbnails = []
@ -61,9 +63,10 @@ class BlinkxIE(InfoExtractor):
elif m['type'] in ('flv', 'mp4'): elif m['type'] in ('flv', 'mp4'):
vcodec = remove_start(m['vcodec'], 'ff') vcodec = remove_start(m['vcodec'], 'ff')
acodec = remove_start(m['acodec'], 'ff') acodec = remove_start(m['acodec'], 'ff')
tbr = (int(m['vbr']) + int(m['abr'])) // 1000
format_id = (u'%s-%sk-%s' % format_id = (u'%s-%sk-%s' %
(vcodec, (vcodec,
(int(m['vbr']) + int(m['abr'])) // 1000, tbr,
m['w'])) m['w']))
formats.append({ formats.append({
'format_id': format_id, 'format_id': format_id,
@ -72,10 +75,12 @@ class BlinkxIE(InfoExtractor):
'acodec': acodec, 'acodec': acodec,
'abr': int(m['abr']) // 1000, 'abr': int(m['abr']) // 1000,
'vbr': int(m['vbr']) // 1000, 'vbr': int(m['vbr']) // 1000,
'tbr': tbr,
'width': int(m['w']), 'width': int(m['w']),
'height': int(m['h']), 'height': int(m['h']),
}) })
formats.sort(key=lambda f: (f['width'], f['vbr'], f['abr']))
self._sort_formats(formats)
return { return {
'id': display_id, 'id': display_id,
@ -83,7 +88,7 @@ class BlinkxIE(InfoExtractor):
'title': data['title'], 'title': data['title'],
'formats': formats, 'formats': formats,
'uploader': data['channel_name'], 'uploader': data['channel_name'],
'upload_date': upload_date, 'upload_date': pload_date,
'description': data.get('description'), 'description': data.get('description'),
'thumbnails': thumbnails, 'thumbnails': thumbnails,
'duration': duration, 'duration': duration,

92
youtube_dl/extractor/bliptv.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import datetime import datetime
import json import json
import re import re
@ -6,10 +8,8 @@ import socket
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
compat_http_client, compat_http_client,
compat_parse_qs,
compat_str, compat_str,
compat_urllib_error, compat_urllib_error,
compat_urllib_parse_urlparse,
compat_urllib_request, compat_urllib_request,
ExtractorError, ExtractorError,
@ -20,42 +20,36 @@ from ..utils import (
class BlipTVIE(InfoExtractor): class BlipTVIE(InfoExtractor):
"""Information extractor for blip.tv""" """Information extractor for blip.tv"""
_VALID_URL = r'^(?:https?://)?(?:www\.)?blip\.tv/((.+/)|(play/)|(api\.swf#))(.+)$' _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv/((.+/)|(play/)|(api\.swf#))(.+)$'
_URL_EXT = r'^.*\.([a-z0-9]+)$'
IE_NAME = u'blip.tv'
_TEST = { _TEST = {
u'url': u'http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352', 'url': 'http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352',
u'file': u'5779306.m4v', 'file': '5779306.mov',
u'md5': u'80baf1ec5c3d2019037c1c707d676b9f', 'md5': 'c6934ad0b6acf2bd920720ec888eb812',
u'info_dict': { 'info_dict': {
u"upload_date": u"20111205", 'upload_date': '20111205',
u"description": u"md5:9bc31f227219cde65e47eeec8d2dc596", 'description': 'md5:9bc31f227219cde65e47eeec8d2dc596',
u"uploader": u"Comic Book Resources - CBR TV", 'uploader': 'Comic Book Resources - CBR TV',
u"title": u"CBR EXCLUSIVE: \"Gotham City Imposters\" Bats VS Jokerz Short 3" 'title': 'CBR EXCLUSIVE: "Gotham City Imposters" Bats VS Jokerz Short 3',
} }
} }
def report_direct_download(self, title): def report_direct_download(self, title):
"""Report information extraction.""" """Report information extraction."""
self.to_screen(u'%s: Direct download detected' % title) self.to_screen('%s: Direct download detected' % title)
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
if mobj is None: if mobj is None:
raise ExtractorError(u'Invalid URL: %s' % url) raise ExtractorError('Invalid URL: %s' % url)
# See https://github.com/rg3/youtube-dl/issues/857 # See https://github.com/rg3/youtube-dl/issues/857
api_mobj = re.match(r'http://a\.blip\.tv/api\.swf#(?P<video_id>[\d\w]+)', url) embed_mobj = re.search(r'^(?:https?://)?(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)([a-zA-Z0-9]+)', url)
if api_mobj is not None: if embed_mobj:
url = 'http://blip.tv/play/g_%s' % api_mobj.group('video_id') info_url = 'http://blip.tv/play/%s.x?p=1' % embed_mobj.group(1)
urlp = compat_urllib_parse_urlparse(url) info_page = self._download_webpage(info_url, embed_mobj.group(1))
if urlp.path.startswith('/play/'): video_id = self._search_regex(r'data-episode-id="(\d+)', info_page, 'video_id')
response = self._request_webpage(url, None, False) return self.url_result('http://blip.tv/a/a-' + video_id, 'BlipTV')
redirecturl = response.geturl()
rurlp = compat_urllib_parse_urlparse(redirecturl)
file_id = compat_parse_qs(rurlp.fragment)['file'][0].rpartition('/')[2]
url = 'http://blip.tv/a/a-' + file_id
return self._real_extract(url)
if '?' in url: if '?' in url:
cchar = '&' cchar = '&'
@ -66,13 +60,13 @@ class BlipTVIE(InfoExtractor):
request.add_header('User-Agent', 'iTunes/10.6.1') request.add_header('User-Agent', 'iTunes/10.6.1')
self.report_extraction(mobj.group(1)) self.report_extraction(mobj.group(1))
urlh = self._request_webpage(request, None, False, urlh = self._request_webpage(request, None, False,
u'unable to download video info webpage') 'unable to download video info webpage')
try: try:
json_code_bytes = urlh.read() json_code_bytes = urlh.read()
json_code = json_code_bytes.decode('utf-8') json_code = json_code_bytes.decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
raise ExtractorError(u'Unable to read video info webpage: %s' % compat_str(err)) raise ExtractorError('Unable to read video info webpage: %s' % compat_str(err))
try: try:
json_data = json.loads(json_code) json_data = json.loads(json_code)
@ -82,32 +76,38 @@ class BlipTVIE(InfoExtractor):
data = json_data data = json_data
upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d') upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
formats = []
if 'additionalMedia' in data: if 'additionalMedia' in data:
formats = sorted(data['additionalMedia'], key=lambda f: int(f['media_height'])) for f in sorted(data['additionalMedia'], key=lambda f: int(f['media_height'])):
best_format = formats[-1] if not int(f['media_width']): # filter m3u8
video_url = best_format['url'] continue
formats.append({
'url': f['url'],
'format_id': f['role'],
'width': int(f['media_width']),
'height': int(f['media_height']),
})
else: else:
video_url = data['media']['url'] formats.append({
umobj = re.match(self._URL_EXT, video_url) 'url': data['media']['url'],
if umobj is None: 'width': int(data['media']['width']),
raise ValueError('Can not determine filename extension') 'height': int(data['media']['height']),
ext = umobj.group(1) })
self._sort_formats(formats)
return { return {
'id': compat_str(data['item_id']), 'id': compat_str(data['item_id']),
'url': video_url,
'uploader': data['display_name'], 'uploader': data['display_name'],
'upload_date': upload_date, 'upload_date': upload_date,
'title': data['title'], 'title': data['title'],
'ext': ext,
'format': data['media']['mimeType'],
'thumbnail': data['thumbnailUrl'], 'thumbnail': data['thumbnailUrl'],
'description': data['description'], 'description': data['description'],
'player_url': data['embedUrl'],
'user_agent': 'iTunes/10.6.1', 'user_agent': 'iTunes/10.6.1',
'formats': formats,
} }
except (ValueError, KeyError) as err: except (ValueError, KeyError) as err:
raise ExtractorError(u'Unable to parse video information: %s' % repr(err)) raise ExtractorError('Unable to parse video information: %s' % repr(err))
class BlipTVUserIE(InfoExtractor): class BlipTVUserIE(InfoExtractor):
@ -115,19 +115,19 @@ class BlipTVUserIE(InfoExtractor):
_VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$' _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$'
_PAGE_SIZE = 12 _PAGE_SIZE = 12
IE_NAME = u'blip.tv:user' IE_NAME = 'blip.tv:user'
def _real_extract(self, url): def _real_extract(self, url):
# Extract username # Extract username
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
if mobj is None: if mobj is None:
raise ExtractorError(u'Invalid URL: %s' % url) raise ExtractorError('Invalid URL: %s' % url)
username = mobj.group(1) username = mobj.group(1)
page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1' page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
page = self._download_webpage(url, username, u'Downloading user page') page = self._download_webpage(url, username, 'Downloading user page')
mobj = re.search(r'data-users-id="([^"]+)"', page) mobj = re.search(r'data-users-id="([^"]+)"', page)
page_base = page_base % mobj.group(1) page_base = page_base % mobj.group(1)
@ -143,7 +143,7 @@ class BlipTVUserIE(InfoExtractor):
while True: while True:
url = page_base + "&page=" + str(pagenum) url = page_base + "&page=" + str(pagenum)
page = self._download_webpage(url, username, page = self._download_webpage(url, username,
u'Downloading video ids from page %d' % pagenum) 'Downloading video ids from page %d' % pagenum)
# Extract video identifiers # Extract video identifiers
ids_in_page = [] ids_in_page = []
@ -165,6 +165,6 @@ class BlipTVUserIE(InfoExtractor):
pagenum += 1 pagenum += 1
urls = [u'http://blip.tv/%s' % video_id for video_id in video_ids] urls = ['http://blip.tv/%s' % video_id for video_id in video_ids]
url_entries = [self.url_result(vurl, 'BlipTV') for vurl in urls] url_entries = [self.url_result(vurl, 'BlipTV') for vurl in urls]
return [self.playlist_result(url_entries, playlist_title = username)] return [self.playlist_result(url_entries, playlist_title = username)]

5
youtube_dl/extractor/bloomberg.py
View File

@ -1,6 +1,7 @@
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from .ooyala import OoyalaIE
class BloombergIE(InfoExtractor): class BloombergIE(InfoExtractor):
@ -23,5 +24,5 @@ class BloombergIE(InfoExtractor):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
name = mobj.group('name') name = mobj.group('name')
webpage = self._download_webpage(url, name) webpage = self._download_webpage(url, name)
ooyala_url = self._og_search_video_url(webpage) ooyala_code = self._search_regex(r'<source src="http://player.ooyala.com/player/[^/]+/([^".]+)', webpage, u'ooyala url')
return self.url_result(ooyala_url, ie='Ooyala') return OoyalaIE._build_url_result(ooyala_code)

83
youtube_dl/extractor/brightcove.py
View File

@ -1,4 +1,5 @@
# encoding: utf-8 # encoding: utf-8
from __future__ import unicode_literals
import re import re
import json import json
@ -13,6 +14,7 @@ from ..utils import (
compat_urllib_request, compat_urllib_request,
ExtractorError, ExtractorError,
unsmuggle_url,
) )
@ -24,47 +26,47 @@ class BrightcoveIE(InfoExtractor):
_TESTS = [ _TESTS = [
{ {
# From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/ # From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/
u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001', 'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
u'file': u'2371591881001.mp4', 'file': '2371591881001.mp4',
u'md5': u'5423e113865d26e40624dce2e4b45d95', 'md5': '5423e113865d26e40624dce2e4b45d95',
u'note': u'Test Brightcove downloads and detection in GenericIE', 'note': 'Test Brightcove downloads and detection in GenericIE',
u'info_dict': { 'info_dict': {
u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”', 'title': 'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
u'uploader': u'8TV', 'uploader': '8TV',
u'description': u'md5:a950cc4285c43e44d763d036710cd9cd', 'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
} }
}, },
{ {
# From http://medianetwork.oracle.com/video/player/1785452137001 # From http://medianetwork.oracle.com/video/player/1785452137001
u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001', 'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
u'file': u'1785452137001.flv', 'file': '1785452137001.flv',
u'info_dict': { 'info_dict': {
u'title': u'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges', 'title': 'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
u'description': u'John Rose speaks at the JVM Language Summit, August 1, 2012.', 'description': 'John Rose speaks at the JVM Language Summit, August 1, 2012.',
u'uploader': u'Oracle', 'uploader': 'Oracle',
}, },
}, },
{ {
# From http://mashable.com/2013/10/26/thermoelectric-bracelet-lets-you-control-your-body-temperature/ # From http://mashable.com/2013/10/26/thermoelectric-bracelet-lets-you-control-your-body-temperature/
u'url': u'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001', 'url': 'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
u'info_dict': { 'info_dict': {
u'id': u'2750934548001', 'id': '2750934548001',
u'ext': u'mp4', 'ext': 'mp4',
u'title': u'This Bracelet Acts as a Personal Thermostat', 'title': 'This Bracelet Acts as a Personal Thermostat',
u'description': u'md5:547b78c64f4112766ccf4e151c20b6a0', 'description': 'md5:547b78c64f4112766ccf4e151c20b6a0',
u'uploader': u'Mashable', 'uploader': 'Mashable',
}, },
}, },
{ {
# test that the default referer works # test that the default referer works
# from http://national.ballet.ca/interact/video/Lost_in_Motion_II/ # from http://national.ballet.ca/interact/video/Lost_in_Motion_II/
u'url': u'http://link.brightcove.com/services/player/bcpid756015033001?bckey=AQ~~,AAAApYJi_Ck~,GxhXCegT1Dp39ilhXuxMJxasUhVNZiil&bctid=2878862109001', 'url': 'http://link.brightcove.com/services/player/bcpid756015033001?bckey=AQ~~,AAAApYJi_Ck~,GxhXCegT1Dp39ilhXuxMJxasUhVNZiil&bctid=2878862109001',
u'info_dict': { 'info_dict': {
u'id': u'2878862109001', 'id': '2878862109001',
u'ext': u'mp4', 'ext': 'mp4',
u'title': u'Lost in Motion II', 'title': 'Lost in Motion II',
u'description': u'md5:363109c02998fee92ec02211bd8000df', 'description': 'md5:363109c02998fee92ec02211bd8000df',
u'uploader': u'National Ballet of Canada', 'uploader': 'National Ballet of Canada',
}, },
}, },
] ]
@ -80,13 +82,13 @@ class BrightcoveIE(InfoExtractor):
object_str = re.sub(r'(<param name="[^"]+" value="[^"]+")>', object_str = re.sub(r'(<param name="[^"]+" value="[^"]+")>',
lambda m: m.group(1) + '/>', object_str) lambda m: m.group(1) + '/>', object_str)
# Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608 # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
object_str = object_str.replace(u'<--', u'<!--') object_str = object_str.replace('<--', '<!--')
object_doc = xml.etree.ElementTree.fromstring(object_str) object_doc = xml.etree.ElementTree.fromstring(object_str)
assert u'BrightcoveExperience' in object_doc.attrib['class'] assert 'BrightcoveExperience' in object_doc.attrib['class']
params = {'flashID': object_doc.attrib['id'], params = {
'playerID': find_xpath_attr(object_doc, './param', 'name', 'playerID').attrib['value'], 'playerID': find_xpath_attr(object_doc, './param', 'name', 'playerID').attrib['value'],
} }
def find_param(name): def find_param(name):
node = find_xpath_attr(object_doc, './param', 'name', name) node = find_xpath_attr(object_doc, './param', 'name', name)
if node is not None: if node is not None:
@ -120,6 +122,8 @@ class BrightcoveIE(InfoExtractor):
return None return None
def _real_extract(self, url): def _real_extract(self, url):
url, smuggled_data = unsmuggle_url(url, {})
# Change the 'videoId' and others field to '@videoPlayer' # Change the 'videoId' and others field to '@videoPlayer'
url = re.sub(r'(?<=[?&])(videoI(d|D)|bctid)', '%40videoPlayer', url) url = re.sub(r'(?<=[?&])(videoI(d|D)|bctid)', '%40videoPlayer', url)
# Change bckey (used by bcove.me urls) to playerKey # Change bckey (used by bcove.me urls) to playerKey
@ -130,9 +134,10 @@ class BrightcoveIE(InfoExtractor):
videoPlayer = query.get('@videoPlayer') videoPlayer = query.get('@videoPlayer')
if videoPlayer: if videoPlayer:
return self._get_video_info(videoPlayer[0], query_str, query, # We set the original url as the default 'Referer' header
# We set the original url as the default 'Referer' header referer = smuggled_data.get('Referer', url)
referer=url) return self._get_video_info(
videoPlayer[0], query_str, query, referer=referer)
else: else:
player_key = query['playerKey'] player_key = query['playerKey']
return self._get_playlist_info(player_key[0]) return self._get_playlist_info(player_key[0])
@ -156,11 +161,11 @@ class BrightcoveIE(InfoExtractor):
def _get_playlist_info(self, player_key): def _get_playlist_info(self, player_key):
playlist_info = self._download_webpage(self._PLAYLIST_URL_TEMPLATE % player_key, playlist_info = self._download_webpage(self._PLAYLIST_URL_TEMPLATE % player_key,
player_key, u'Downloading playlist information') player_key, 'Downloading playlist information')
json_data = json.loads(playlist_info) json_data = json.loads(playlist_info)
if 'videoList' not in json_data: if 'videoList' not in json_data:
raise ExtractorError(u'Empty playlist') raise ExtractorError('Empty playlist')
playlist_info = json_data['videoList'] playlist_info = json_data['videoList']
videos = [self._extract_video_info(video_info) for video_info in playlist_info['mediaCollectionDTO']['videoDTOs']] videos = [self._extract_video_info(video_info) for video_info in playlist_info['mediaCollectionDTO']['videoDTOs']]
@ -189,5 +194,5 @@ class BrightcoveIE(InfoExtractor):
'url': video_info['FLVFullLengthURL'], 'url': video_info['FLVFullLengthURL'],
}) })
else: else:
raise ExtractorError(u'Unable to extract video url for %s' % info['id']) raise ExtractorError('Unable to extract video url for %s' % info['id'])
return info return info

40
youtube_dl/extractor/c56.py
View File

@ -1,21 +1,21 @@
# coding: utf-8 # coding: utf-8
from __future__ import unicode_literals
import re import re
import json import json
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import determine_ext
class C56IE(InfoExtractor): class C56IE(InfoExtractor):
_VALID_URL = r'https?://((www|player)\.)?56\.com/(.+?/)?(v_|(play_album.+-))(?P<textid>.+?)\.(html|swf)' _VALID_URL = r'https?://((www|player)\.)?56\.com/(.+?/)?(v_|(play_album.+-))(?P<textid>.+?)\.(html|swf)'
IE_NAME = u'56.com' IE_NAME = '56.com'
_TEST = {
_TEST ={ 'url': 'http://www.56.com/u39/v_OTM0NDA3MTY.html',
u'url': u'http://www.56.com/u39/v_OTM0NDA3MTY.html', 'file': '93440716.flv',
u'file': u'93440716.flv', 'md5': 'e59995ac63d0457783ea05f93f12a866',
u'md5': u'e59995ac63d0457783ea05f93f12a866', 'info_dict': {
u'info_dict': { 'title': '网事知多少 第32期：车怒',
u'title': u'网事知多少 第32期：车怒',
}, },
} }
@ -23,14 +23,18 @@ class C56IE(InfoExtractor):
mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE) mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
text_id = mobj.group('textid') text_id = mobj.group('textid')
info_page = self._download_webpage('http://vxml.56.com/json/%s/' % text_id, info_page = self._download_webpage('http://vxml.56.com/json/%s/' % text_id,
text_id, u'Downloading video info') text_id, 'Downloading video info')
info = json.loads(info_page)['info'] info = json.loads(info_page)['info']
best_format = sorted(info['rfiles'], key=lambda f: int(f['filesize']))[-1] formats = [{
video_url = best_format['url'] 'format_id': f['type'],
'filesize': int(f['filesize']),
'url': f['url']
} for f in info['rfiles']]
self._sort_formats(formats)
return {'id': info['vid'], return {
'title': info['Subject'], 'id': info['vid'],
'url': video_url, 'title': info['Subject'],
'ext': determine_ext(video_url), 'formats': formats,
'thumbnail': info.get('bimg') or info.get('img'), 'thumbnail': info.get('bimg') or info.get('img'),
} }

20
youtube_dl/extractor/channel9.py
View File

@ -76,14 +76,18 @@ class Channel9IE(InfoExtractor):
</div>)? # File size part may be missing </div>)? # File size part may be missing
''' '''
# Extract known formats # Extract known formats
formats = [{'url': x.group('url'), formats = [{
'format_id': x.group('quality'), 'url': x.group('url'),
'format_note': x.group('note'), 'format_id': x.group('quality'),
'format': '%s (%s)' % (x.group('quality'), x.group('note')), 'format_note': x.group('note'),
'filesize': self._restore_bytes(x.group('filesize')), # File size is approximate 'format': u'%s (%s)' % (x.group('quality'), x.group('note')),
} for x in list(re.finditer(FORMAT_REGEX, html)) if x.group('quality') in self._known_formats] 'filesize': self._restore_bytes(x.group('filesize')), # File size is approximate
# Sort according to known formats list 'preference': self._known_formats.index(x.group('quality')),
formats.sort(key=lambda fmt: self._known_formats.index(fmt['format_id'])) 'vcodec': 'none' if x.group('note') == 'Audio only' else None,
} for x in list(re.finditer(FORMAT_REGEX, html)) if x.group('quality') in self._known_formats]
self._sort_formats(formats)
return formats return formats
def _extract_title(self, html): def _extract_title(self, html):

19
youtube_dl/extractor/cmt.py Normal file
View File

@ -0,0 +1,19 @@
from .mtv import MTVIE
class CMTIE(MTVIE):
IE_NAME = u'cmt.com'
_VALID_URL = r'https?://www\.cmt\.com/videos/.+?/(?P<videoid>[^/]+)\.jhtml'
_FEED_URL = 'http://www.cmt.com/sitewide/apps/player/embed/rss/'
_TESTS = [
{
u'url': u'http://www.cmt.com/videos/garth-brooks/989124/the-call-featuring-trisha-yearwood.jhtml#artist=30061',
u'md5': u'e6b7ef3c4c45bbfae88061799bbba6c2',
u'info_dict': {
u'id': u'989124',
u'ext': u'mp4',
u'title': u'Garth Brooks - "The Call (featuring Trisha Yearwood)"',
u'description': u'Blame It All On My Roots',
},
},
]

33
youtube_dl/extractor/cnn.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
@ -12,23 +14,24 @@ class CNNIE(InfoExtractor):
(?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn|(?=&)))''' (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn|(?=&)))'''
_TESTS = [{ _TESTS = [{
u'url': u'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn', 'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
u'file': u'sports_2013_06_09_nadal-1-on-1.cnn.mp4', 'file': 'sports_2013_06_09_nadal-1-on-1.cnn.mp4',
u'md5': u'3e6121ea48df7e2259fe73a0628605c4', 'md5': '3e6121ea48df7e2259fe73a0628605c4',
u'info_dict': { 'info_dict': {
u'title': u'Nadal wins 8th French Open title', 'title': 'Nadal wins 8th French Open title',
u'description': u'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.', 'description': 'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.',
u'duration': 135, 'duration': 135,
u'upload_date': u'20130609', 'upload_date': '20130609',
}, },
}, },
{ {
u"url": u"http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29", "url": "http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29",
u"file": u"us_2013_08_21_sot-student-gives-epic-speech.georgia-institute-of-technology.mp4", "file": "us_2013_08_21_sot-student-gives-epic-speech.georgia-institute-of-technology.mp4",
u"md5": u"b5cc60c60a3477d185af8f19a2a26f4e", "md5": "b5cc60c60a3477d185af8f19a2a26f4e",
u"info_dict": { "info_dict": {
u"title": "Student's epic speech stuns new freshmen", "title": "Student's epic speech stuns new freshmen",
u"description": "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"" "description": "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
"upload_date": "20130821",
} }
}] }]
@ -36,7 +39,7 @@ class CNNIE(InfoExtractor):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
path = mobj.group('path') path = mobj.group('path')
page_title = mobj.group('title') page_title = mobj.group('title')
info_url = u'http://cnn.com/video/data/3.0/%s/index.xml' % path info_url = 'http://cnn.com/video/data/3.0/%s/index.xml' % path
info = self._download_xml(info_url, page_title) info = self._download_xml(info_url, page_title)
formats = [] formats = []

106
youtube_dl/extractor/collegehumor.py
View File

@ -1,82 +1,68 @@
from __future__ import unicode_literals
import json
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import (
compat_urllib_parse_urlparse,
determine_ext,
ExtractorError,
)
class CollegeHumorIE(InfoExtractor): class CollegeHumorIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/(video|embed|e)/(?P<videoid>[0-9]+)/?(?P<shorttitle>.*)$' _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/(video|embed|e)/(?P<videoid>[0-9]+)/?(?P<shorttitle>.*)$'
_TESTS = [{ _TESTS = [{
u'url': u'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe', 'url': 'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe',
u'file': u'6902724.mp4', 'file': '6902724.mp4',
u'md5': u'1264c12ad95dca142a9f0bf7968105a0', 'md5': 'dcc0f5c1c8be98dc33889a191f4c26bd',
u'info_dict': { 'info_dict': {
u'title': u'Comic-Con Cosplay Catastrophe', 'title': 'Comic-Con Cosplay Catastrophe',
u'description': u'Fans get creative this year at San Diego. Too creative. And yes, that\'s really Joss Whedon.', 'description': 'Fans get creative this year at San Diego. Too',
'age_limit': 13,
}, },
}, },
{ {
u'url': u'http://www.collegehumor.com/video/3505939/font-conference', 'url': 'http://www.collegehumor.com/video/3505939/font-conference',
u'file': u'3505939.mp4', 'file': '3505939.mp4',
u'md5': u'c51ca16b82bb456a4397987791a835f5', 'md5': '72fa701d8ef38664a4dbb9e2ab721816',
u'info_dict': { 'info_dict': {
u'title': u'Font Conference', 'title': 'Font Conference',
u'description': u'This video wasn\'t long enough, so we made it double-spaced.', 'description': 'This video wasn\'t long enough, so we made it double-spaced.',
'age_limit': 10,
}, },
}] }]
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
if mobj is None:
raise ExtractorError(u'Invalid URL: %s' % url)
video_id = mobj.group('videoid') video_id = mobj.group('videoid')
info = { jsonUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id + '.json'
'id': video_id, data = json.loads(self._download_webpage(
'uploader': None, jsonUrl, video_id, 'Downloading info JSON'))
'upload_date': None, vdata = data['video']
}
self.report_extraction(video_id) AGE_LIMITS = {'nc17': 18, 'r': 18, 'pg13': 13, 'pg': 10, 'g': 0}
xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id rating = vdata.get('rating')
mdoc = self._download_xml(xmlUrl, video_id, if rating:
u'Downloading info XML', age_limit = AGE_LIMITS.get(rating.lower())
u'Unable to download video info XML')
try:
videoNode = mdoc.findall('./video')[0]
youtubeIdNode = videoNode.find('./youtubeID')
if youtubeIdNode is not None:
return self.url_result(youtubeIdNode.text, 'Youtube')
info['description'] = videoNode.findall('./description')[0].text
info['title'] = videoNode.findall('./caption')[0].text
info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
next_url = videoNode.findall('./file')[0].text
except IndexError:
raise ExtractorError(u'Invalid metadata XML file')
if next_url.endswith(u'manifest.f4m'):
manifest_url = next_url + '?hdcore=2.10.3'
adoc = self._download_xml(manifest_url, video_id,
u'Downloading XML manifest',
u'Unable to download video info XML')
try:
video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text
except IndexError:
raise ExtractorError(u'Invalid manifest file')
url_pr = compat_urllib_parse_urlparse(info['thumbnail'])
info['url'] = url_pr.scheme + '://' + url_pr.netloc + video_id[:-2].replace('.csmil','').replace(',','')
info['ext'] = 'mp4'
else: else:
# Old-style direct links age_limit = None # None = No idea
info['url'] = next_url
info['ext'] = determine_ext(info['url'])
return info PREFS = {'high_quality': 2, 'low_quality': 0}
formats = []
for format_key in ('mp4', 'webm'):
for qname, qurl in vdata[format_key].items():
formats.append({
'format_id': format_key + '_' + qname,
'url': qurl,
'format': format_key,
'preference': PREFS.get(qname),
})
self._sort_formats(formats)
return {
'id': video_id,
'title': vdata['title'],
'description': vdata.get('description'),
'thumbnail': vdata.get('thumbnail'),
'formats': formats,
'age_limit': age_limit,
}

4
youtube_dl/extractor/comedycentral.py
View File

@ -12,7 +12,9 @@ from ..utils import (
class ComedyCentralIE(MTVServicesInfoExtractor): class ComedyCentralIE(MTVServicesInfoExtractor):
_VALID_URL = r'https?://(?:www.)?comedycentral.com/(video-clips|episodes|cc-studios)/(?P<title>.*)' _VALID_URL = r'''(?x)https?://(?:www.)?comedycentral.com/
(video-clips|episodes|cc-studios|video-collections)
/(?P<title>.*)'''
_FEED_URL = u'http://comedycentral.com/feeds/mrss/' _FEED_URL = u'http://comedycentral.com/feeds/mrss/'
_TEST = { _TEST = {

28
youtube_dl/extractor/common.py
View File

@ -1,4 +1,6 @@
import base64 import base64
import hashlib
import json
import os import os
import re import re
import socket import socket
@ -51,7 +53,8 @@ class InfoExtractor(object):
Calculated from the format_id, width, height. Calculated from the format_id, width, height.
and format_note fields if missing. and format_note fields if missing.
* format_id A short description of the format * format_id A short description of the format
("mp4_h264_opus" or "19") ("mp4_h264_opus" or "19").
Technically optional, but strongly recommended.
* format_note Additional info about the format * format_note Additional info about the format
("3D" or "DASH video") ("3D" or "DASH video")
* width Width of the video, if known * width Width of the video, if known
@ -68,7 +71,12 @@ class InfoExtractor(object):
download, lower-case. download, lower-case.
"http", "https", "rtsp", "rtmp" or so. "http", "https", "rtsp", "rtmp" or so.
* preference Order number of this format. If this field is * preference Order number of this format. If this field is
present, the formats get sorted by this field. present and not None, the formats get sorted
by this field.
-1 for default (order by other properties),
-2 or smaller for less than default.
* quality Order number of the video quality of this
format, irrespective of the file format.
-1 for default (order by other properties), -1 for default (order by other properties),
-2 or smaller for less than default. -2 or smaller for less than default.
url: Final video URL. url: Final video URL.
@ -227,6 +235,9 @@ class InfoExtractor(object):
url = url_or_request.get_full_url() url = url_or_request.get_full_url()
except AttributeError: except AttributeError:
url = url_or_request url = url_or_request
if len(url) > 200:
h = hashlib.md5(url).hexdigest()
url = url[:200 - len(h)] + h
raw_filename = ('%s_%s.dump' % (video_id, url)) raw_filename = ('%s_%s.dump' % (video_id, url))
filename = sanitize_filename(raw_filename, restricted=True) filename = sanitize_filename(raw_filename, restricted=True)
self.to_screen(u'Saving request to ' + filename) self.to_screen(u'Saving request to ' + filename)
@ -254,6 +265,15 @@ class InfoExtractor(object):
xml_string = transform_source(xml_string) xml_string = transform_source(xml_string)
return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8')) return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
def _download_json(self, url_or_request, video_id,
note=u'Downloading JSON metadata',
errnote=u'Unable to download JSON metadata'):
json_string = self._download_webpage(url_or_request, video_id, note, errnote)
try:
return json.loads(json_string)
except ValueError as ve:
raise ExtractorError('Failed to download JSON', cause=ve)
def report_warning(self, msg, video_id=None): def report_warning(self, msg, video_id=None):
idstr = u'' if video_id is None else u'%s: ' % video_id idstr = u'' if video_id is None else u'%s: ' % video_id
self._downloader.report_warning( self._downloader.report_warning(
@ -376,7 +396,7 @@ class InfoExtractor(object):
@staticmethod @staticmethod
def _og_regexes(prop): def _og_regexes(prop):
content_re = r'content=(?:"([^>]+?)"|\'(.+?)\')' content_re = r'content=(?:"([^>]+?)"|\'(.+?)\')'
property_re = r'property=[\'"]og:%s[\'"]' % re.escape(prop) property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
template = r'<meta[^>]+?%s[^>]+?%s' template = r'<meta[^>]+?%s[^>]+?%s'
return [ return [
template % (property_re, content_re), template % (property_re, content_re),
@ -481,9 +501,11 @@ class InfoExtractor(object):
return ( return (
preference, preference,
f.get('quality') if f.get('quality') is not None else -1,
f.get('height') if f.get('height') is not None else -1, f.get('height') if f.get('height') is not None else -1,
f.get('width') if f.get('width') is not None else -1, f.get('width') if f.get('width') is not None else -1,
ext_preference, ext_preference,
f.get('tbr') if f.get('tbr') is not None else -1,
f.get('vbr') if f.get('vbr') is not None else -1, f.get('vbr') if f.get('vbr') is not None else -1,
f.get('abr') if f.get('abr') is not None else -1, f.get('abr') if f.get('abr') is not None else -1,
audio_ext_preference, audio_ext_preference,

74
youtube_dl/extractor/condenast.py
View File

@ -1,4 +1,5 @@
# coding: utf-8 # coding: utf-8
from __future__ import unicode_literals
import re import re
import json import json
@ -20,30 +21,31 @@ class CondeNastIE(InfoExtractor):
# The keys are the supported sites and the values are the name to be shown # The keys are the supported sites and the values are the name to be shown
# to the user and in the extractor description. # to the user and in the extractor description.
_SITES = {'wired': u'WIRED', _SITES = {
'gq': u'GQ', 'wired': 'WIRED',
'vogue': u'Vogue', 'gq': 'GQ',
'glamour': u'Glamour', 'vogue': 'Vogue',
'wmagazine': u'W Magazine', 'glamour': 'Glamour',
'vanityfair': u'Vanity Fair', 'wmagazine': 'W Magazine',
} 'vanityfair': 'Vanity Fair',
}
_VALID_URL = r'http://(video|www).(?P<site>%s).com/(?P<type>watch|series|video)/(?P<id>.+)' % '|'.join(_SITES.keys()) _VALID_URL = r'http://(video|www).(?P<site>%s).com/(?P<type>watch|series|video)/(?P<id>.+)' % '|'.join(_SITES.keys())
IE_DESC = u'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values())) IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
_TEST = { _TEST = {
u'url': u'http://video.wired.com/watch/3d-printed-speakers-lit-with-led', 'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',
u'file': u'5171b343c2b4c00dd0c1ccb3.mp4', 'file': '5171b343c2b4c00dd0c1ccb3.mp4',
u'md5': u'1921f713ed48aabd715691f774c451f7', 'md5': '1921f713ed48aabd715691f774c451f7',
u'info_dict': { 'info_dict': {
u'title': u'3D Printed Speakers Lit With LED', 'title': '3D Printed Speakers Lit With LED',
u'description': u'Check out these beautiful 3D printed LED speakers. You can\'t actually buy them, but LumiGeek is working on a board that will let you make you\'re own.', 'description': 'Check out these beautiful 3D printed LED speakers. You can\'t actually buy them, but LumiGeek is working on a board that will let you make you\'re own.',
} }
} }
def _extract_series(self, url, webpage): def _extract_series(self, url, webpage):
title = self._html_search_regex(r'<div class="cne-series-info">.*?<h1>(.+?)</h1>', title = self._html_search_regex(r'<div class="cne-series-info">.*?<h1>(.+?)</h1>',
webpage, u'series title', flags=re.DOTALL) webpage, 'series title', flags=re.DOTALL)
url_object = compat_urllib_parse_urlparse(url) url_object = compat_urllib_parse_urlparse(url)
base_url = '%s://%s' % (url_object.scheme, url_object.netloc) base_url = '%s://%s' % (url_object.scheme, url_object.netloc)
m_paths = re.finditer(r'<p class="cne-thumb-title">.*?<a href="(/watch/.+?)["\?]', m_paths = re.finditer(r'<p class="cne-thumb-title">.*?<a href="(/watch/.+?)["\?]',
@ -57,39 +59,41 @@ class CondeNastIE(InfoExtractor):
description = self._html_search_regex([r'<div class="cne-video-description">(.+?)</div>', description = self._html_search_regex([r'<div class="cne-video-description">(.+?)</div>',
r'<div class="video-post-content">(.+?)</div>', r'<div class="video-post-content">(.+?)</div>',
], ],
webpage, u'description', webpage, 'description',
fatal=False, flags=re.DOTALL) fatal=False, flags=re.DOTALL)
params = self._search_regex(r'var params = {(.+?)}[;,]', webpage, params = self._search_regex(r'var params = {(.+?)}[;,]', webpage,
u'player params', flags=re.DOTALL) 'player params', flags=re.DOTALL)
video_id = self._search_regex(r'videoId: [\'"](.+?)[\'"]', params, u'video id') video_id = self._search_regex(r'videoId: [\'"](.+?)[\'"]', params, 'video id')
player_id = self._search_regex(r'playerId: [\'"](.+?)[\'"]', params, u'player id') player_id = self._search_regex(r'playerId: [\'"](.+?)[\'"]', params, 'player id')
target = self._search_regex(r'target: [\'"](.+?)[\'"]', params, u'target') target = self._search_regex(r'target: [\'"](.+?)[\'"]', params, 'target')
data = compat_urllib_parse.urlencode({'videoId': video_id, data = compat_urllib_parse.urlencode({'videoId': video_id,
'playerId': player_id, 'playerId': player_id,
'target': target, 'target': target,
}) })
base_info_url = self._search_regex(r'url = [\'"](.+?)[\'"][,;]', base_info_url = self._search_regex(r'url = [\'"](.+?)[\'"][,;]',
webpage, u'base info url', webpage, 'base info url',
default='http://player.cnevids.com/player/loader.js?') default='http://player.cnevids.com/player/loader.js?')
info_url = base_info_url + data info_url = base_info_url + data
info_page = self._download_webpage(info_url, video_id, info_page = self._download_webpage(info_url, video_id,
u'Downloading video info') 'Downloading video info')
video_info = self._search_regex(r'var video = ({.+?});', info_page, u'video info') video_info = self._search_regex(r'var video = ({.+?});', info_page, 'video info')
video_info = json.loads(video_info) video_info = json.loads(video_info)
def _formats_sort_key(f): formats = [{
type_ord = 1 if f['type'] == 'video/mp4' else 0 'format_id': '%s-%s' % (fdata['type'].split('/')[-1], fdata['quality']),
quality_ord = 1 if f['quality'] == 'high' else 0 'url': fdata['src'],
return (quality_ord, type_ord) 'ext': fdata['type'].split('/')[-1],
best_format = sorted(video_info['sources'][0], key=_formats_sort_key)[-1] 'quality': 1 if fdata['quality'] == 'high' else 0,
} for fdata in video_info['sources'][0]]
self._sort_formats(formats)
return {'id': video_id, return {
'url': best_format['src'], 'id': video_id,
'ext': best_format['type'].split('/')[-1], 'formats': formats,
'title': video_info['title'], 'title': video_info['title'],
'thumbnail': video_info['poster_frame'], 'thumbnail': video_info['poster_frame'],
'description': description, 'description': description,
} }
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)

57
youtube_dl/extractor/cspan.py
View File

@ -1,20 +1,25 @@
from __future__ import unicode_literals
import json
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
compat_urllib_parse, unescapeHTML,
) )
class CSpanIE(InfoExtractor): class CSpanIE(InfoExtractor):
_VALID_URL = r'http://www\.c-spanvideo\.org/program/(.*)' _VALID_URL = r'http://www\.c-spanvideo\.org/program/(.*)'
IE_DESC = 'C-SPAN'
_TEST = { _TEST = {
u'url': u'http://www.c-spanvideo.org/program/HolderonV', 'url': 'http://www.c-spanvideo.org/program/HolderonV',
u'file': u'315139.flv', 'file': '315139.mp4',
u'md5': u'74a623266956f69e4df0068ab6c80fe4', 'md5': '8e44ce11f0f725527daccc453f553eb0',
u'info_dict': { 'info_dict': {
u"title": u"Attorney General Eric Holder on Voting Rights Act Decision" 'title': 'Attorney General Eric Holder on Voting Rights Act Decision',
'description': 'Attorney General Eric Holder spoke to reporters following the Supreme Court decision in [Shelby County v. Holder] in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced until Congress established new guidelines for review.',
}, },
u'skip': u'Requires rtmpdump'
} }
def _real_extract(self, url): def _real_extract(self, url):
@ -22,30 +27,22 @@ class CSpanIE(InfoExtractor):
prog_name = mobj.group(1) prog_name = mobj.group(1)
webpage = self._download_webpage(url, prog_name) webpage = self._download_webpage(url, prog_name)
video_id = self._search_regex(r'programid=(.*?)&', webpage, 'video id') video_id = self._search_regex(r'programid=(.*?)&', webpage, 'video id')
data = compat_urllib_parse.urlencode({'programid': video_id,
'dynamic':'1'})
info_url = 'http://www.c-spanvideo.org/common/services/flashXml.php?' + data
video_info = self._download_webpage(info_url, video_id, u'Downloading video info')
self.report_extraction(video_id) title = self._html_search_regex(
r'<!-- title -->\n\s*<h1[^>]*>(.*?)</h1>', webpage, 'title')
description = self._og_search_description(webpage)
title = self._html_search_regex(r'<string name="title">(.*?)</string>', info_url = 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=program&id=' + video_id
video_info, 'title') data_json = self._download_webpage(
description = self._html_search_regex(r'<meta (?:property="og:|name=")description" content="(.*?)"', info_url, video_id, 'Downloading video info')
webpage, 'description', data = json.loads(data_json)
flags=re.MULTILINE|re.DOTALL)
url = self._search_regex(r'<string name="URL">(.*?)</string>', url = unescapeHTML(data['video']['files'][0]['path']['#text'])
video_info, 'video url')
url = url.replace('$(protocol)', 'rtmp').replace('$(port)', '443')
path = self._search_regex(r'<string name="path">(.*?)</string>',
video_info, 'rtmp play path')
return {'id': video_id, return {
'title': title, 'id': video_id,
'ext': 'flv', 'title': title,
'url': url, 'url': url,
'play_path': path, 'description': description,
'description': description, 'thumbnail': self._og_search_thumbnail(webpage),
'thumbnail': self._og_search_thumbnail(webpage), }
}

11
youtube_dl/extractor/defense.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import re import re
import json import json
@ -5,15 +7,14 @@ from .common import InfoExtractor
class DefenseGouvFrIE(InfoExtractor): class DefenseGouvFrIE(InfoExtractor):
_IE_NAME = 'defense.gouv.fr' IE_NAME = 'defense.gouv.fr'
_VALID_URL = (r'http://.*?\.defense\.gouv\.fr/layout/set/' _VALID_URL = (r'http://.*?\.defense\.gouv\.fr/layout/set/'
r'ligthboxvideo/base-de-medias/webtv/(.*)') r'ligthboxvideo/base-de-medias/webtv/(.*)')
_TEST = { _TEST = {
u'url': (u'http://www.defense.gouv.fr/layout/set/ligthboxvideo/' 'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
u'base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1'), 'file': '11213.mp4',
u'file': u'11213.mp4', 'md5': '75bba6124da7e63d2d60b5244ec9430c',
u'md5': u'75bba6124da7e63d2d60b5244ec9430c',
"info_dict": { "info_dict": {
"title": "attaque-chimique-syrienne-du-21-aout-2013-1" "title": "attaque-chimique-syrienne-du-21-aout-2013-1"
} }

6
youtube_dl/extractor/dreisat.py
View File

@ -10,11 +10,11 @@ from ..utils import (
class DreiSatIE(InfoExtractor): class DreiSatIE(InfoExtractor):
IE_NAME = '3sat' IE_NAME = '3sat'
_VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/index\.php\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$' _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
_TEST = { _TEST = {
u"url": u"http://www.3sat.de/mediathek/index.php?obj=36983", u"url": u"http://www.3sat.de/mediathek/index.php?obj=36983",
u'file': u'36983.webm', u'file': u'36983.mp4',
u'md5': u'57c97d0469d71cf874f6815aa2b7c944', u'md5': u'9dcfe344732808dbfcc901537973c922',
u'info_dict': { u'info_dict': {
u"title": u"Kaffeeland Schweiz", u"title": u"Kaffeeland Schweiz",
u"description": u"Über 80 Kaffeeröstereien liefern in der Schweiz das Getränk, in das das Land so vernarrt ist: Mehr als 1000 Tassen trinkt ein Schweizer pro Jahr. SCHWEIZWEIT nimmt die Kaffeekultur unter die...", u"description": u"Über 80 Kaffeeröstereien liefern in der Schweiz das Getränk, in das das Land so vernarrt ist: Mehr als 1000 Tassen trinkt ein Schweizer pro Jahr. SCHWEIZWEIT nimmt die Kaffeekultur unter die...",

69
youtube_dl/extractor/everyonesmixtape.py Normal file
View File

@ -0,0 +1,69 @@
from __future__ import unicode_literals
import re
from .common import InfoExtractor
from ..utils import (
compat_urllib_request,
ExtractorError,
)
class EveryonesMixtapeIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?everyonesmixtape\.com/#/mix/(?P<id>[0-9a-zA-Z]+)(?:/(?P<songnr>[0-9]))?$'
_TEST = {
'url': 'http://everyonesmixtape.com/#/mix/m7m0jJAbMQi/5',
'file': '5bfseWNmlds.mp4',
"info_dict": {
"title": "Passion Pit - \"Sleepyhead\" (Official Music Video)",
"uploader": "FKR.TV",
"uploader_id": "frenchkissrecords",
"description": "Music video for \"Sleepyhead\" from Passion Pit's debut EP Chunk Of Change.\nBuy on iTunes: https://itunes.apple.com/us/album/chunk-of-change-ep/id300087641\n\nDirected by The Wilderness.\n\nhttp://www.passionpitmusic.com\nhttp://www.frenchkissrecords.com",
"upload_date": "20081015"
},
'params': {
'skip_download': True, # This is simply YouTube
}
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
playlist_id = mobj.group('id')
pllist_url = 'http://everyonesmixtape.com/mixtape.php?a=getMixes&u=-1&linked=%s&explore=' % playlist_id
pllist_req = compat_urllib_request.Request(pllist_url)
pllist_req.add_header('X-Requested-With', 'XMLHttpRequest')
playlist_list = self._download_json(
pllist_req, playlist_id, note='Downloading playlist metadata')
try:
playlist_no = next(playlist['id']
for playlist in playlist_list
if playlist['code'] == playlist_id)
except StopIteration:
raise ExtractorError('Playlist id not found')
pl_url = 'http://everyonesmixtape.com/mixtape.php?a=getMix&id=%s&userId=null&code=' % playlist_no
pl_req = compat_urllib_request.Request(pl_url)
pl_req.add_header('X-Requested-With', 'XMLHttpRequest')
playlist = self._download_json(
pl_req, playlist_id, note='Downloading playlist info')
entries = [{
'_type': 'url',
'url': t['url'],
'title': t['title'],
} for t in playlist['tracks']]
if mobj.group('songnr'):
songnr = int(mobj.group('songnr')) - 1
return entries[songnr]
playlist_title = playlist['mixData']['name']
return {
'_type': 'playlist',
'id': playlist_id,
'title': playlist_title,
'entries': entries,
}

22
youtube_dl/extractor/flickr.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
@ -11,13 +13,13 @@ class FlickrIE(InfoExtractor):
"""Information Extractor for Flickr videos""" """Information Extractor for Flickr videos"""
_VALID_URL = r'(?:https?://)?(?:www\.|secure\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*' _VALID_URL = r'(?:https?://)?(?:www\.|secure\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
_TEST = { _TEST = {
u'url': u'http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/', 'url': 'http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/',
u'file': u'5645318632.mp4', 'file': '5645318632.mp4',
u'md5': u'6fdc01adbc89d72fc9c4f15b4a4ba87b', 'md5': '6fdc01adbc89d72fc9c4f15b4a4ba87b',
u'info_dict': { 'info_dict': {
u"description": u"Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.", "description": "Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.",
u"uploader_id": u"forestwander-nature-pictures", "uploader_id": "forestwander-nature-pictures",
u"title": u"Dark Hollow Waterfalls" "title": "Dark Hollow Waterfalls"
} }
} }
@ -29,13 +31,13 @@ class FlickrIE(InfoExtractor):
webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
webpage = self._download_webpage(webpage_url, video_id) webpage = self._download_webpage(webpage_url, video_id)
secret = self._search_regex(r"photo_secret: '(\w+)'", webpage, u'secret') secret = self._search_regex(r"photo_secret: '(\w+)'", webpage, 'secret')
first_url = 'https://secure.flickr.com/apps/video/video_mtl_xml.gne?v=x&photo_id=' + video_id + '&secret=' + secret + '&bitrate=700&target=_self' first_url = 'https://secure.flickr.com/apps/video/video_mtl_xml.gne?v=x&photo_id=' + video_id + '&secret=' + secret + '&bitrate=700&target=_self'
first_xml = self._download_webpage(first_url, video_id, 'Downloading first data webpage') first_xml = self._download_webpage(first_url, video_id, 'Downloading first data webpage')
node_id = self._html_search_regex(r'<Item id="id">(\d+-\d+)</Item>', node_id = self._html_search_regex(r'<Item id="id">(\d+-\d+)</Item>',
first_xml, u'node_id') first_xml, 'node_id')
second_url = 'https://secure.flickr.com/video_playlist.gne?node_id=' + node_id + '&tech=flash&mode=playlist&bitrate=700&secret=' + secret + '&rd=video.yahoo.com&noad=1' second_url = 'https://secure.flickr.com/video_playlist.gne?node_id=' + node_id + '&tech=flash&mode=playlist&bitrate=700&secret=' + secret + '&rd=video.yahoo.com&noad=1'
second_xml = self._download_webpage(second_url, video_id, 'Downloading second data webpage') second_xml = self._download_webpage(second_url, video_id, 'Downloading second data webpage')
@ -44,7 +46,7 @@ class FlickrIE(InfoExtractor):
mobj = re.search(r'<STREAM APP="(.+?)" FULLPATH="(.+?)"', second_xml) mobj = re.search(r'<STREAM APP="(.+?)" FULLPATH="(.+?)"', second_xml)
if mobj is None: if mobj is None:
raise ExtractorError(u'Unable to extract video url') raise ExtractorError('Unable to extract video url')
video_url = mobj.group(1) + unescapeHTML(mobj.group(2)) video_url = mobj.group(1) + unescapeHTML(mobj.group(2))
return [{ return [{

38
youtube_dl/extractor/franceinter.py Normal file
View File

@ -0,0 +1,38 @@
# coding: utf-8
from __future__ import unicode_literals
import re
from .common import InfoExtractor
class FranceInterIE(InfoExtractor):
_VALID_URL = r'http://(?:www\.)?franceinter\.fr/player/reecouter\?play=(?P<id>[0-9]{6})'
_TEST = {
'url': 'http://www.franceinter.fr/player/reecouter?play=793962',
'file': '793962.mp3',
'md5': '4764932e466e6f6c79c317d2e74f6884',
"info_dict": {
"title": "L’Histoire dans les jeux vidéo",
},
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
webpage = self._download_webpage(url, video_id)
title = self._html_search_regex(
r'<span class="roll_overflow">(.*?)</span></h1>', webpage, 'title')
path = self._search_regex(
r'&urlAOD=(.*?)&startTime', webpage, 'video url')
video_url = 'http://www.franceinter.fr/' + path
return {
'id': video_id,
'formats': [{
'url': video_url,
'vcodec': 'none',
}],
'title': title,
}

26
youtube_dl/extractor/francetv.py
View File

@ -191,3 +191,29 @@ class GenerationQuoiIE(InfoExtractor):
info = json.loads(info_json) info = json.loads(info_json)
return self.url_result('http://www.dailymotion.com/video/%s' % info['id'], return self.url_result('http://www.dailymotion.com/video/%s' % info['id'],
ie='Dailymotion') ie='Dailymotion')
class CultureboxIE(FranceTVBaseInfoExtractor):
IE_NAME = u'culturebox.francetvinfo.fr'
_VALID_URL = r'https?://culturebox\.francetvinfo\.fr/(?P<name>.*?)(\?|$)'
_TEST = {
u'url': u'http://culturebox.francetvinfo.fr/einstein-on-the-beach-au-theatre-du-chatelet-146813',
u'info_dict': {
u'id': u'EV_6785',
u'ext': u'mp4',
u'title': u'Einstein on the beach au Théâtre du Châtelet',
u'description': u'md5:9ce2888b1efefc617b5e58b3f6200eeb',
},
u'params': {
# m3u8 download
u'skip_download': True,
},
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
name = mobj.group('name')
webpage = self._download_webpage(url, name)
video_id = self._search_regex(r'"http://videos\.francetv\.fr/video/(.*?)"', webpage, u'video id')
return self._extract_video(video_id)

14
youtube_dl/extractor/gamespot.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import re import re
import json import json
@ -13,12 +15,12 @@ from ..utils import (
class GameSpotIE(InfoExtractor): class GameSpotIE(InfoExtractor):
_VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<page_id>\d+)/?' _VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<page_id>\d+)/?'
_TEST = { _TEST = {
u"url": u"http://www.gamespot.com/arma-iii/videos/arma-iii-community-guide-sitrep-i-6410818/", "url": "http://www.gamespot.com/arma-iii/videos/arma-iii-community-guide-sitrep-i-6410818/",
u"file": u"gs-2300-6410818.mp4", "file": "gs-2300-6410818.mp4",
u"md5": u"b2a30deaa8654fcccd43713a6b6a4825", "md5": "b2a30deaa8654fcccd43713a6b6a4825",
u"info_dict": { "info_dict": {
u"title": u"Arma 3 - Community Guide: SITREP I", "title": "Arma 3 - Community Guide: SITREP I",
u'description': u'Check out this video where some of the basics of Arma 3 is explained.', 'description': 'Check out this video where some of the basics of Arma 3 is explained.',
} }
} }

164
youtube_dl/extractor/generic.py
View File

@ -1,9 +1,12 @@
# encoding: utf-8 # encoding: utf-8
from __future__ import unicode_literals
import os import os
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from .youtube import YoutubeIE
from ..utils import ( from ..utils import (
compat_urllib_error, compat_urllib_error,
compat_urllib_parse, compat_urllib_parse,
@ -22,78 +25,78 @@ from .ooyala import OoyalaIE
class GenericIE(InfoExtractor): class GenericIE(InfoExtractor):
IE_DESC = u'Generic downloader that works on some sites' IE_DESC = 'Generic downloader that works on some sites'
_VALID_URL = r'.*' _VALID_URL = r'.*'
IE_NAME = u'generic' IE_NAME = 'generic'
_TESTS = [ _TESTS = [
{ {
u'url': u'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html', 'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
u'file': u'13601338388002.mp4', 'file': '13601338388002.mp4',
u'md5': u'6e15c93721d7ec9e9ca3fdbf07982cfd', 'md5': '6e15c93721d7ec9e9ca3fdbf07982cfd',
u'info_dict': { 'info_dict': {
u"uploader": u"www.hodiho.fr", 'uploader': 'www.hodiho.fr',
u"title": u"R\u00e9gis plante sa Jeep" 'title': 'R\u00e9gis plante sa Jeep',
} }
}, },
# embedded vimeo video # embedded vimeo video
{ {
u'add_ie': ['Vimeo'], 'add_ie': ['Vimeo'],
u'url': u'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references', 'url': 'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references',
u'file': u'22444065.mp4', 'file': '22444065.mp4',
u'md5': u'2903896e23df39722c33f015af0666e2', 'md5': '2903896e23df39722c33f015af0666e2',
u'info_dict': { 'info_dict': {
u'title': u'ACCU 2011: Move Semantics,Perfect Forwarding, and Rvalue references- Scott Meyers- 13/04/2011', 'title': 'ACCU 2011: Move Semantics,Perfect Forwarding, and Rvalue references- Scott Meyers- 13/04/2011',
u"uploader_id": u"skillsmatter", 'uploader_id': 'skillsmatter',
u"uploader": u"Skills Matter", 'uploader': 'Skills Matter',
} }
}, },
# bandcamp page with custom domain # bandcamp page with custom domain
{ {
u'add_ie': ['Bandcamp'], 'add_ie': ['Bandcamp'],
u'url': u'http://bronyrock.com/track/the-pony-mash', 'url': 'http://bronyrock.com/track/the-pony-mash',
u'file': u'3235767654.mp3', 'file': '3235767654.mp3',
u'info_dict': { 'info_dict': {
u'title': u'The Pony Mash', 'title': 'The Pony Mash',
u'uploader': u'M_Pallante', 'uploader': 'M_Pallante',
}, },
u'skip': u'There is a limit of 200 free downloads / month for the test song', 'skip': 'There is a limit of 200 free downloads / month for the test song',
}, },
# embedded brightcove video # embedded brightcove video
# it also tests brightcove videos that need to set the 'Referer' in the # it also tests brightcove videos that need to set the 'Referer' in the
# http requests # http requests
{ {
u'add_ie': ['Brightcove'], 'add_ie': ['Brightcove'],
u'url': u'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/', 'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
u'info_dict': { 'info_dict': {
u'id': u'2765128793001', 'id': '2765128793001',
u'ext': u'mp4', 'ext': 'mp4',
u'title': u'Le cours de bourse : l’analyse technique', 'title': 'Le cours de bourse : l’analyse technique',
u'description': u'md5:7e9ad046e968cb2d1114004aba466fd9', 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
u'uploader': u'BFM BUSINESS', 'uploader': 'BFM BUSINESS',
}, },
u'params': { 'params': {
u'skip_download': True, 'skip_download': True,
}, },
}, },
# Direct link to a video # Direct link to a video
{ {
u'url': u'http://media.w3.org/2010/05/sintel/trailer.mp4', 'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
u'file': u'trailer.mp4', 'file': 'trailer.mp4',
u'md5': u'67d406c2bcb6af27fa886f31aa934bbe', 'md5': '67d406c2bcb6af27fa886f31aa934bbe',
u'info_dict': { 'info_dict': {
u'id': u'trailer', 'id': 'trailer',
u'title': u'trailer', 'title': 'trailer',
u'upload_date': u'20100513', 'upload_date': '20100513',
} }
}, },
# ooyala video # ooyala video
{ {
u'url': u'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219', 'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
u'md5': u'5644c6ca5d5782c1d0d350dad9bd840c', 'md5': '5644c6ca5d5782c1d0d350dad9bd840c',
u'info_dict': { 'info_dict': {
u'id': u'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ', 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
u'ext': u'mp4', 'ext': 'mp4',
u'title': u'2cc213299525360.mov', #that's what we get 'title': '2cc213299525360.mov', #that's what we get
}, },
}, },
] ]
@ -101,12 +104,12 @@ class GenericIE(InfoExtractor):
def report_download_webpage(self, video_id): def report_download_webpage(self, video_id):
"""Report webpage download.""" """Report webpage download."""
if not self._downloader.params.get('test', False): if not self._downloader.params.get('test', False):
self._downloader.report_warning(u'Falling back on generic information extractor.') self._downloader.report_warning('Falling back on generic information extractor.')
super(GenericIE, self).report_download_webpage(video_id) super(GenericIE, self).report_download_webpage(video_id)
def report_following_redirect(self, new_url): def report_following_redirect(self, new_url):
"""Report information extraction.""" """Report information extraction."""
self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url) self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
def _send_head(self, url): def _send_head(self, url):
"""Check if it is a redirect, like url shorteners, in case return the new url.""" """Check if it is a redirect, like url shorteners, in case return the new url."""
@ -152,7 +155,7 @@ class GenericIE(InfoExtractor):
response = opener.open(HEADRequest(url)) response = opener.open(HEADRequest(url))
if response is None: if response is None:
raise ExtractorError(u'Invalid URL protocol') raise ExtractorError('Invalid URL protocol')
return response return response
def _real_extract(self, url): def _real_extract(self, url):
@ -162,6 +165,8 @@ class GenericIE(InfoExtractor):
return self.url_result('http://' + url) return self.url_result('http://' + url)
video_id = os.path.splitext(url.split('/')[-1])[0] video_id = os.path.splitext(url.split('/')[-1])[0]
self.to_screen('%s: Requesting header' % video_id)
try: try:
response = self._send_head(url) response = self._send_head(url)
@ -184,7 +189,7 @@ class GenericIE(InfoExtractor):
'formats': [{ 'formats': [{
'format_id': m.group('format_id'), 'format_id': m.group('format_id'),
'url': url, 'url': url,
'vcodec': u'none' if m.group('type') == 'audio' else None 'vcodec': 'none' if m.group('type') == 'audio' else None
}], }],
'upload_date': upload_date, 'upload_date': upload_date,
} }
@ -198,7 +203,7 @@ class GenericIE(InfoExtractor):
except ValueError: except ValueError:
# since this is the last-resort InfoExtractor, if # since this is the last-resort InfoExtractor, if
# this error is thrown, it'll be thrown here # this error is thrown, it'll be thrown here
raise ExtractorError(u'Failed to download URL: %s' % url) raise ExtractorError('Failed to download URL: %s' % url)
self.report_extraction(video_id) self.report_extraction(video_id)
@ -209,22 +214,23 @@ class GenericIE(InfoExtractor):
# Video Title - Tagline | Site Name # Video Title - Tagline | Site Name
# and so on and so forth; it's just not practical # and so on and so forth; it's just not practical
video_title = self._html_search_regex( video_title = self._html_search_regex(
r'(?s)<title>(.*?)</title>', webpage, u'video title', r'(?s)<title>(.*?)</title>', webpage, 'video title',
default=u'video') default='video')
# video uploader is domain name # video uploader is domain name
video_uploader = self._search_regex( video_uploader = self._search_regex(
r'^(?:https?://)?([^/]*)/.*', url, u'video uploader') r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
# Look for BrightCove: # Look for BrightCove:
bc_url = BrightcoveIE._extract_brightcove_url(webpage) bc_url = BrightcoveIE._extract_brightcove_url(webpage)
if bc_url is not None: if bc_url is not None:
self.to_screen(u'Brightcove video detected.') self.to_screen('Brightcove video detected.')
return self.url_result(bc_url, 'Brightcove') surl = smuggle_url(bc_url, {'Referer': url})
return self.url_result(surl, 'Brightcove')
# Look for embedded (iframe) Vimeo player # Look for embedded (iframe) Vimeo player
mobj = re.search( mobj = re.search(
r'<iframe[^>]+?src="(https?://player.vimeo.com/video/.+?)"', webpage) r'<iframe[^>]+?src="((?:https?:)?//player.vimeo.com/video/.+?)"', webpage)
if mobj: if mobj:
player_url = unescapeHTML(mobj.group(1)) player_url = unescapeHTML(mobj.group(1))
surl = smuggle_url(player_url, {'Referer': url}) surl = smuggle_url(player_url, {'Referer': url})
@ -271,16 +277,12 @@ class GenericIE(InfoExtractor):
} }
# Look for embedded blip.tv player # Look for embedded blip.tv player
mobj = re.search(r'<meta\s[^>]*https?://api.blip.tv/\w+/redirect/\w+/(\d+)', webpage) mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
if mobj: if mobj:
return self.url_result('http://blip.tv/seo/-'+mobj.group(1), 'BlipTV') return self.url_result('http://blip.tv/a/a-'+mobj.group(1), 'BlipTV')
mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*https?://(?:\w+\.)?blip.tv/(?:play/|api\.swf#)([a-zA-Z0-9]+)', webpage) mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9]+)', webpage)
if mobj: if mobj:
player_url = 'http://blip.tv/play/%s.x?p=1' % mobj.group(1) return self.url_result(mobj.group(1), 'BlipTV')
player_page = self._download_webpage(player_url, mobj.group(1))
blip_video_id = self._search_regex(r'data-episode-id="(\d+)', player_page, u'blip_video_id', fatal=False)
if blip_video_id:
return self.url_result('http://blip.tv/seo/-'+blip_video_id, 'BlipTV')
# Look for Bandcamp pages with custom domain # Look for Bandcamp pages with custom domain
mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage) mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
@ -301,18 +303,32 @@ class GenericIE(InfoExtractor):
return OoyalaIE._build_url_result(mobj.group(1)) return OoyalaIE._build_url_result(mobj.group(1))
# Look for Aparat videos # Look for Aparat videos
mobj = re.search(r'<iframe src="(http://www.aparat.com/video/[^"]+)"', webpage) mobj = re.search(r'<iframe src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
if mobj is not None: if mobj is not None:
return self.url_result(mobj.group(1), 'Aparat') return self.url_result(mobj.group(1), 'Aparat')
# Look for MPORA videos
mobj = re.search(r'<iframe .*?src="(http://mpora\.com/videos/[^"]+)"', webpage)
if mobj is not None:
return self.url_result(mobj.group(1), 'Mpora')
# Look for embedded Novamov player
mobj = re.search(
r'<iframe[^>]+?src=(["\'])(?P<url>http://(?:(?:embed|www)\.)?novamov\.com/embed\.php.+?)\1', webpage)
if mobj is not None:
return self.url_result(mobj.group('url'), 'Novamov')
# Start with something easy: JW Player in SWFObject # Start with something easy: JW Player in SWFObject
mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
if mobj is None:
# Look for gorilla-vid style embedding
mobj = re.search(r'(?s)jw_plugins.*?file:\s*["\'](.*?)["\']', webpage)
if mobj is None: if mobj is None:
# Broaden the search a little bit # Broaden the search a little bit
mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage) mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
if mobj is None: if mobj is None:
# Broaden the search a little bit: JWPlayer JS loader # Broaden the search a little bit: JWPlayer JS loader
mobj = re.search(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http[^\'"]*)', webpage) mobj = re.search(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage)
if mobj is None: if mobj is None:
# Try to find twitter cards info # Try to find twitter cards info
mobj = re.search(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage) mobj = re.search(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage)
@ -327,23 +343,27 @@ class GenericIE(InfoExtractor):
# HTML5 video # HTML5 video
mobj = re.search(r'<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage, flags=re.DOTALL) mobj = re.search(r'<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage, flags=re.DOTALL)
if mobj is None: if mobj is None:
raise ExtractorError(u'Unsupported URL: %s' % url) raise ExtractorError('Unsupported URL: %s' % url)
# It's possible that one of the regexes # It's possible that one of the regexes
# matched, but returned an empty group: # matched, but returned an empty group:
if mobj.group(1) is None: if mobj.group(1) is None:
raise ExtractorError(u'Did not find a valid video URL at %s' % url) raise ExtractorError('Did not find a valid video URL at %s' % url)
video_url = mobj.group(1) video_url = mobj.group(1)
video_url = compat_urlparse.urljoin(url, video_url) video_url = compat_urlparse.urljoin(url, video_url)
video_id = compat_urllib_parse.unquote(os.path.basename(video_url)) video_id = compat_urllib_parse.unquote(os.path.basename(video_url))
# Sometimes, jwplayer extraction will result in a YouTube URL
if YoutubeIE.suitable(video_url):
return self.url_result(video_url, 'Youtube')
# here's a fun little line of code for you: # here's a fun little line of code for you:
video_id = os.path.splitext(video_id)[0] video_id = os.path.splitext(video_id)[0]
return { return {
'id': video_id, 'id': video_id,
'url': video_url, 'url': video_url,
'uploader': video_uploader, 'uploader': video_uploader,
'title': video_title, 'title': video_title,
} }

54
youtube_dl/extractor/imdb.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import re import re
import json import json
@ -9,18 +11,18 @@ from ..utils import (
class ImdbIE(InfoExtractor): class ImdbIE(InfoExtractor):
IE_NAME = u'imdb' IE_NAME = 'imdb'
IE_DESC = u'Internet Movie Database trailers' IE_DESC = 'Internet Movie Database trailers'
_VALID_URL = r'http://(?:www|m)\.imdb\.com/video/imdb/vi(?P<id>\d+)' _VALID_URL = r'http://(?:www|m)\.imdb\.com/video/imdb/vi(?P<id>\d+)'
_TEST = { _TEST = {
u'url': u'http://www.imdb.com/video/imdb/vi2524815897', 'url': 'http://www.imdb.com/video/imdb/vi2524815897',
u'md5': u'9f34fa777ade3a6e57a054fdbcb3a068', 'md5': '9f34fa777ade3a6e57a054fdbcb3a068',
u'info_dict': { 'info_dict': {
u'id': u'2524815897', 'id': '2524815897',
u'ext': u'mp4', 'ext': 'mp4',
u'title': u'Ice Age: Continental Drift Trailer (No. 2) - IMDb', 'title': 'Ice Age: Continental Drift Trailer (No. 2) - IMDb',
u'description': u'md5:9061c2219254e5d14e03c25c98e96a81', 'description': 'md5:9061c2219254e5d14e03c25c98e96a81',
} }
} }
@ -37,10 +39,10 @@ class ImdbIE(InfoExtractor):
f_path = f_path.strip() f_path = f_path.strip()
format_page = self._download_webpage( format_page = self._download_webpage(
compat_urlparse.urljoin(url, f_path), compat_urlparse.urljoin(url, f_path),
u'Downloading info for %s format' % f_id) 'Downloading info for %s format' % f_id)
json_data = self._search_regex( json_data = self._search_regex(
r'<script[^>]+class="imdb-player-data"[^>]*?>(.*?)</script>', r'<script[^>]+class="imdb-player-data"[^>]*?>(.*?)</script>',
format_page, u'json data', flags=re.DOTALL) format_page, 'json data', flags=re.DOTALL)
info = json.loads(json_data) info = json.loads(json_data)
format_info = info['videoPlayerObject']['video'] format_info = info['videoPlayerObject']['video']
formats.append({ formats.append({
@ -55,3 +57,33 @@ class ImdbIE(InfoExtractor):
'description': descr, 'description': descr,
'thumbnail': format_info['slate'], 'thumbnail': format_info['slate'],
} }
class ImdbListIE(InfoExtractor):
IE_NAME = 'imdb:list'
IE_DESC = 'Internet Movie Database lists'
_VALID_URL = r'http://www\.imdb\.com/list/(?P<id>[\da-zA-Z_-]{11})'
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
list_id = mobj.group('id')
# RSS XML is sometimes malformed
rss = self._download_webpage('http://rss.imdb.com/list/%s' % list_id, list_id, 'Downloading list RSS')
list_title = self._html_search_regex(r'<title>(.*?)</title>', rss, 'list title')
# Export is independent of actual author_id, but returns 404 if no author_id is provided.
# However, passing dummy author_id seems to be enough.
csv = self._download_webpage('http://www.imdb.com/list/export?list_id=%s&author_id=ur00000000' % list_id,
list_id, 'Downloading list CSV')
entries = []
for item in csv.split('\n')[1:]:
cols = item.split(',')
if len(cols) < 2:
continue
item_id = cols[1][1:-1]
if item_id.startswith('vi'):
entries.append(self.url_result('http://www.imdb.com/video/imdb/%s' % item_id, 'Imdb'))
return self.playlist_result(entries, list_id, list_title)

2
youtube_dl/extractor/ina.py
View File

@ -7,7 +7,7 @@ class InaIE(InfoExtractor):
"""Information Extractor for Ina.fr""" """Information Extractor for Ina.fr"""
_VALID_URL = r'(?:http://)?(?:www\.)?ina\.fr/video/(?P<id>I?[A-F0-9]+)/.*' _VALID_URL = r'(?:http://)?(?:www\.)?ina\.fr/video/(?P<id>I?[A-F0-9]+)/.*'
_TEST = { _TEST = {
u'url': u'www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html', u'url': u'http://www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html',
u'file': u'I12055569.mp4', u'file': u'I12055569.mp4',
u'md5': u'a667021bf2b41f8dc6049479d9bb38a3', u'md5': u'a667021bf2b41f8dc6049479d9bb38a3',
u'info_dict': { u'info_dict': {

13
youtube_dl/extractor/internetvideoarchive.py
View File

@ -5,7 +5,6 @@ from ..utils import (
compat_urlparse, compat_urlparse,
compat_urllib_parse, compat_urllib_parse,
xpath_with_ns, xpath_with_ns,
determine_ext,
) )
@ -63,13 +62,17 @@ class InternetVideoArchiveIE(InfoExtractor):
for content in item.findall(_bp('media:group/media:content')): for content in item.findall(_bp('media:group/media:content')):
attr = content.attrib attr = content.attrib
f_url = attr['url'] f_url = attr['url']
width = int(attr['width'])
bitrate = int(attr['bitrate'])
format_id = '%d-%dk' % (width, bitrate)
formats.append({ formats.append({
'format_id': format_id,
'url': f_url, 'url': f_url,
'ext': determine_ext(f_url), 'width': width,
'width': int(attr['width']), 'tbr': bitrate,
'bitrate': int(attr['bitrate']),
}) })
formats = sorted(formats, key=lambda f: f['bitrate'])
self._sort_formats(formats)
return { return {
'id': video_id, 'id': video_id,

16
youtube_dl/extractor/ivi.py
View File

@ -84,14 +84,16 @@ class IviIE(InfoExtractor):
result = video_json[u'result'] result = video_json[u'result']
formats = [{'url': x[u'url'], formats = [{
'format_id': x[u'content_format'] 'url': x[u'url'],
} for x in result[u'files'] if x[u'content_format'] in self._known_formats] 'format_id': x[u'content_format'],
formats.sort(key=lambda fmt: self._known_formats.index(fmt['format_id'])) 'preference': self._known_formats.index(x[u'content_format']),
} for x in result[u'files'] if x[u'content_format'] in self._known_formats]
if len(formats) == 0: self._sort_formats(formats)
self._downloader.report_warning(u'No media links available for %s' % video_id)
return if not formats:
raise ExtractorError(u'No media links available for %s' % video_id)
duration = result[u'duration'] duration = result[u'duration']
compilation = result[u'compilation'] compilation = result[u'compilation']

73
youtube_dl/extractor/jpopsukitv.py Normal file
View File

@ -0,0 +1,73 @@
# coding=utf-8
from __future__ import unicode_literals
import re
from .common import InfoExtractor
from ..utils import (
int_or_none,
unified_strdate,
)
class JpopsukiIE(InfoExtractor):
IE_NAME = 'jpopsuki.tv'
_VALID_URL = r'https?://(?:www\.)?jpopsuki\.tv/video/(.*?)/(?P<id>\S+)'
_TEST = {
'url': 'http://www.jpopsuki.tv/video/ayumi-hamasaki---evolution/00be659d23b0b40508169cdee4545771',
'md5': '88018c0c1a9b1387940e90ec9e7e198e',
'file': '00be659d23b0b40508169cdee4545771.mp4',
'info_dict': {
'id': '00be659d23b0b40508169cdee4545771',
'title': 'ayumi hamasaki - evolution',
'description': 'Release date: 2001.01.31\r\n浜崎あゆみ - evolution',
'thumbnail': 'http://www.jpopsuki.tv/cache/89722c74d2a2ebe58bcac65321c115b2.jpg',
'uploader': 'plama_chan',
'uploader_id': '404',
'upload_date': '20121101'
}
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
webpage = self._download_webpage(url, video_id)
video_url = 'http://www.jpopsuki.tv' + self._html_search_regex(
r'<source src="(.*?)" type', webpage, 'video url')
video_title = self._og_search_title(webpage)
description = self._og_search_description(webpage)
thumbnail = self._og_search_thumbnail(webpage)
uploader = self._html_search_regex(
r'<li>from: <a href="/user/view/user/(.*?)/uid/',
webpage, 'video uploader', fatal=False)
uploader_id = self._html_search_regex(
r'<li>from: <a href="/user/view/user/\S*?/uid/(\d*)',
webpage, 'video uploader_id', fatal=False)
upload_date = self._html_search_regex(
r'<li>uploaded: (.*?)</li>', webpage, 'video upload_date',
fatal=False)
if upload_date is not None:
upload_date = unified_strdate(upload_date)
view_count_str = self._html_search_regex(
r'<li>Hits: ([0-9]+?)</li>', webpage, 'video view_count',
fatal=False)
comment_count_str = self._html_search_regex(
r'<h2>([0-9]+?) comments</h2>', webpage, 'video comment_count',
fatal=False)
return {
'id': video_id,
'url': video_url,
'title': video_title,
'description': description,
'thumbnail': thumbnail,
'uploader': uploader,
'uploader_id': uploader_id,
'upload_date': upload_date,
'view_count': int_or_none(view_count_str),
'comment_count': int_or_none(comment_count_str),
}

40
youtube_dl/extractor/kankan.py
View File

@ -1,21 +1,24 @@
from __future__ import unicode_literals
import re import re
import hashlib import hashlib
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import determine_ext
_md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest() _md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
class KankanIE(InfoExtractor): class KankanIE(InfoExtractor):
_VALID_URL = r'https?://(?:.*?\.)?kankan\.com/.+?/(?P<id>\d+)\.shtml' _VALID_URL = r'https?://(?:.*?\.)?kankan\.com/.+?/(?P<id>\d+)\.shtml'
_TEST = { _TEST = {
u'url': u'http://yinyue.kankan.com/vod/48/48863.shtml', 'url': 'http://yinyue.kankan.com/vod/48/48863.shtml',
u'file': u'48863.flv', 'file': '48863.flv',
u'md5': u'29aca1e47ae68fc28804aca89f29507e', 'md5': '29aca1e47ae68fc28804aca89f29507e',
u'info_dict': { 'info_dict': {
u'title': u'Ready To Go', 'title': 'Ready To Go',
}, },
'skip': 'Only available from China',
} }
def _real_extract(self, url): def _real_extract(self, url):
@ -23,22 +26,23 @@ class KankanIE(InfoExtractor):
video_id = mobj.group('id') video_id = mobj.group('id')
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
title = self._search_regex(r'(?:G_TITLE=|G_MOVIE_TITLE = )[\'"](.+?)[\'"]', webpage, u'video title') title = self._search_regex(r'(?:G_TITLE=|G_MOVIE_TITLE = )[\'"](.+?)[\'"]', webpage, 'video title')
surls = re.search(r'surls:\[\'.+?\'\]|lurl:\'.+?\.flv\'', webpage).group(0) surls = re.search(r'surls:\[\'.+?\'\]|lurl:\'.+?\.flv\'', webpage).group(0)
gcids = re.findall(r"http://.+?/.+?/(.+?)/", surls) gcids = re.findall(r"http://.+?/.+?/(.+?)/", surls)
gcid = gcids[-1] gcid = gcids[-1]
video_info_page = self._download_webpage('http://p2s.cl.kankan.com/getCdnresource_flv?gcid=%s' % gcid, info_url = 'http://p2s.cl.kankan.com/getCdnresource_flv?gcid=%s' % gcid
video_id, u'Downloading video url info') video_info_page = self._download_webpage(
ip = self._search_regex(r'ip:"(.+?)"', video_info_page, u'video url ip') info_url, video_id, 'Downloading video url info')
path = self._search_regex(r'path:"(.+?)"', video_info_page, u'video url path') ip = self._search_regex(r'ip:"(.+?)"', video_info_page, 'video url ip')
param1 = self._search_regex(r'param1:(\d+)', video_info_page, u'param1') path = self._search_regex(r'path:"(.+?)"', video_info_page, 'video url path')
param2 = self._search_regex(r'param2:(\d+)', video_info_page, u'param2') param1 = self._search_regex(r'param1:(\d+)', video_info_page, 'param1')
param2 = self._search_regex(r'param2:(\d+)', video_info_page, 'param2')
key = _md5('xl_mp43651' + param1 + param2) key = _md5('xl_mp43651' + param1 + param2)
video_url = 'http://%s%s?key=%s&key1=%s' % (ip, path, key, param2) video_url = 'http://%s%s?key=%s&key1=%s' % (ip, path, key, param2)
return {'id': video_id, return {
'title': title, 'id': video_id,
'url': video_url, 'title': title,
'ext': determine_ext(video_url), 'url': video_url,
} }

71
youtube_dl/extractor/khanacademy.py Normal file
View File

@ -0,0 +1,71 @@
from __future__ import unicode_literals
import re
from .common import InfoExtractor
from ..utils import (
unified_strdate,
)
class KhanAcademyIE(InfoExtractor):
_VALID_URL = r'^https?://(?:www\.)?khanacademy\.org/(?P<key>[^/]+)/(?:[^/]+/){,2}(?P<id>[^?#/]+)(?:$|[?#])'
IE_NAME = 'KhanAcademy'
_TEST = {
'url': 'http://www.khanacademy.org/video/one-time-pad',
'file': 'one-time-pad.mp4',
'md5': '7021db7f2d47d4fff89b13177cb1e8f4',
'info_dict': {
'title': 'The one-time pad',
'description': 'The perfect cipher',
'duration': 176,
'uploader': 'Brit Cruise',
'upload_date': '20120411',
}
}
def _real_extract(self, url):
m = re.match(self._VALID_URL, url)
video_id = m.group('id')
if m.group('key') == 'video':
data = self._download_json(
'http://api.khanacademy.org/api/v1/videos/' + video_id,
video_id, 'Downloading video info')
upload_date = unified_strdate(data['date_added'])
uploader = ', '.join(data['author_names'])
return {
'_type': 'url_transparent',
'url': data['url'],
'id': video_id,
'title': data['title'],
'thumbnail': data['image_url'],
'duration': data['duration'],
'description': data['description'],
'uploader': uploader,
'upload_date': upload_date,
}
else:
# topic
data = self._download_json(
'http://api.khanacademy.org/api/v1/topic/' + video_id,
video_id, 'Downloading topic info')
entries = [
{
'_type': 'url',
'url': c['url'],
'id': c['id'],
'title': c['title'],
}
for c in data['children'] if c['kind'] in ('Video', 'Topic')]
return {
'_type': 'playlist',
'id': video_id,
'title': data['title'],
'description': data['description'],
'entries': entries,
}

201
youtube_dl/extractor/lynda.py Normal file
View File

@ -0,0 +1,201 @@
from __future__ import unicode_literals
import re
import json
from .subtitles import SubtitlesInfoExtractor
from .common import InfoExtractor
from ..utils import (
compat_urllib_parse,
compat_urllib_request,
ExtractorError
)
class LyndaIE(SubtitlesInfoExtractor):
IE_NAME = 'lynda'
IE_DESC = 'lynda.com videos'
_VALID_URL = r'https?://www\.lynda\.com/[^/]+/[^/]+/\d+/(\d+)-\d\.html'
_LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
_NETRC_MACHINE = 'lynda'
_SUCCESSFUL_LOGIN_REGEX = r'<a href="https://www.lynda.com/home/userAccount/ChangeContactInfo.aspx" data-qa="eyebrow_account_menu">My account'
_TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
_TEST = {
'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
'file': '114408.mp4',
'md5': 'ecfc6862da89489161fb9cd5f5a6fac1',
'info_dict': {
'title': 'Using the exercise files',
'duration': 68
}
}
def _real_initialize(self):
self._login()
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group(1)
page = self._download_webpage('http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id,
video_id, 'Downloading video JSON')
video_json = json.loads(page)
if 'Status' in video_json:
raise ExtractorError('lynda returned error: %s' % video_json['Message'], expected=True)
if video_json['HasAccess'] is False:
raise ExtractorError('Video %s is only available for members. ' % video_id + self.ACCOUNT_CREDENTIALS_HINT, expected=True)
video_id = video_json['ID']
duration = video_json['DurationInSeconds']
title = video_json['Title']
formats = [{'url': fmt['Url'],
'ext': fmt['Extension'],
'width': fmt['Width'],
'height': fmt['Height'],
'filesize': fmt['FileSize'],
'format_id': str(fmt['Resolution'])
} for fmt in video_json['Formats']]
self._sort_formats(formats)
if self._downloader.params.get('listsubtitles', False):
self._list_available_subtitles(video_id, page)
return
subtitles = self._fix_subtitles(self.extract_subtitles(video_id, page))
return {
'id': video_id,
'title': title,
'duration': duration,
'subtitles': subtitles,
'formats': formats
}
def _login(self):
(username, password) = self._get_login_info()
if username is None:
return
login_form = {
'username': username,
'password': password,
'remember': 'false',
'stayPut': 'false'
}
request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
# Not (yet) logged in
m = re.search(r'loginResultJson = \'(?P<json>[^\']+)\';', login_page)
if m is not None:
response = m.group('json')
response_json = json.loads(response)
state = response_json['state']
if state == 'notlogged':
raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
# This is when we get popup:
# > You're already logged in to lynda.com on two devices.
# > If you log in here, we'll log you out of another device.
# So, we need to confirm this.
if state == 'conflicted':
confirm_form = {
'username': '',
'password': '',
'resolve': 'true',
'remember': 'false',
'stayPut': 'false',
}
request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form))
login_page = self._download_webpage(request, None, note='Confirming log in and log out from another device')
if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
raise ExtractorError('Unable to log in')
def _fix_subtitles(self, subtitles):
if subtitles is None:
return subtitles # subtitles not requested
fixed_subtitles = {}
for k, v in subtitles.items():
subs = json.loads(v)
if len(subs) == 0:
continue
srt = ''
for pos in range(0, len(subs) - 1):
seq_current = subs[pos]
m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode'])
if m_current is None:
continue
seq_next = subs[pos + 1]
m_next = re.match(self._TIMECODE_REGEX, seq_next['Timecode'])
if m_next is None:
continue
appear_time = m_current.group('timecode')
disappear_time = m_next.group('timecode')
text = seq_current['Caption']
srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
if srt:
fixed_subtitles[k] = srt
return fixed_subtitles
def _get_available_subtitles(self, video_id, webpage):
url = 'http://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
sub = self._download_webpage(url, None, note=False)
sub_json = json.loads(sub)
return {'en': url} if len(sub_json) > 0 else {}
class LyndaCourseIE(InfoExtractor):
IE_NAME = 'lynda:course'
IE_DESC = 'lynda.com online courses'
# Course link equals to welcome/introduction video link of same course
# We will recognize it as course link
_VALID_URL = r'https?://(?:www|m)\.lynda\.com/(?P<coursepath>[^/]+/[^/]+/(?P<courseid>\d+))-\d\.html'
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
course_path = mobj.group('coursepath')
course_id = mobj.group('courseid')
page = self._download_webpage('http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
course_id, 'Downloading course JSON')
course_json = json.loads(page)
if 'Status' in course_json and course_json['Status'] == 'NotFound':
raise ExtractorError('Course %s does not exist' % course_id, expected=True)
unaccessible_videos = 0
videos = []
(username, _) = self._get_login_info()
for chapter in course_json['Chapters']:
for video in chapter['Videos']:
if username is None and video['HasAccess'] is False:
unaccessible_videos += 1
continue
videos.append(video['ID'])
if unaccessible_videos > 0:
self._downloader.report_warning('%s videos are only available for members and will not be downloaded. '
% unaccessible_videos + LyndaIE.ACCOUNT_CREDENTIALS_HINT)
entries = [
self.url_result('http://www.lynda.com/%s/%s-4.html' %
(course_path, video_id),
'Lynda')
for video_id in videos]
course_title = course_json['Title']
return self.playlist_result(entries, course_id, course_title)

43
youtube_dl/extractor/macgamestore.py Normal file
View File

@ -0,0 +1,43 @@
from __future__ import unicode_literals
import re
from .common import InfoExtractor
from ..utils import ExtractorError
class MacGameStoreIE(InfoExtractor):
IE_NAME = 'macgamestore'
IE_DESC = 'MacGameStore trailers'
_VALID_URL = r'https?://www\.macgamestore\.com/mediaviewer\.php\?trailer=(?P<id>\d+)'
_TEST = {
'url': 'http://www.macgamestore.com/mediaviewer.php?trailer=2450',
'file': '2450.m4v',
'md5': '8649b8ea684b6666b4c5be736ecddc61',
'info_dict': {
'title': 'Crow',
}
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
webpage = self._download_webpage(url, video_id, 'Downloading trailer page')
if re.search(r'>Missing Media<', webpage) is not None:
raise ExtractorError('Trailer %s does not exist' % video_id, expected=True)
video_title = self._html_search_regex(
r'<title>MacGameStore: (.*?) Trailer</title>', webpage, 'title')
video_url = self._html_search_regex(
r'(?s)<div\s+id="video-player".*?href="([^"]+)"\s*>',
webpage, 'video URL')
return {
'id': video_id,
'url': video_url,
'title': video_title
}

23
youtube_dl/extractor/metacritic.py
View File

@ -1,5 +1,6 @@
from __future__ import unicode_literals
import re import re
import operator
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
@ -11,12 +12,12 @@ class MetacriticIE(InfoExtractor):
_VALID_URL = r'https?://www\.metacritic\.com/.+?/trailers/(?P<id>\d+)' _VALID_URL = r'https?://www\.metacritic\.com/.+?/trailers/(?P<id>\d+)'
_TEST = { _TEST = {
u'url': u'http://www.metacritic.com/game/playstation-4/infamous-second-son/trailers/3698222', 'url': 'http://www.metacritic.com/game/playstation-4/infamous-second-son/trailers/3698222',
u'file': u'3698222.mp4', 'file': '3698222.mp4',
u'info_dict': { 'info_dict': {
u'title': u'inFamous: Second Son - inSide Sucker Punch: Smoke & Mirrors', 'title': 'inFamous: Second Son - inSide Sucker Punch: Smoke & Mirrors',
u'description': u'Take a peak behind-the-scenes to see how Sucker Punch brings smoke into the universe of inFAMOUS Second Son on the PS4.', 'description': 'Take a peak behind-the-scenes to see how Sucker Punch brings smoke into the universe of inFAMOUS Second Son on the PS4.',
u'duration': 221, 'duration': 221,
}, },
} }
@ -26,7 +27,7 @@ class MetacriticIE(InfoExtractor):
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
# The xml is not well formatted, there are raw '&' # The xml is not well formatted, there are raw '&'
info = self._download_xml('http://www.metacritic.com/video_data?video=' + video_id, info = self._download_xml('http://www.metacritic.com/video_data?video=' + video_id,
video_id, u'Downloading info xml', transform_source=fix_xml_all_ampersand) video_id, 'Downloading info xml', transform_source=fix_xml_all_ampersand)
clip = next(c for c in info.findall('playList/clip') if c.find('id').text == video_id) clip = next(c for c in info.findall('playList/clip') if c.find('id').text == video_id)
formats = [] formats = []
@ -37,12 +38,12 @@ class MetacriticIE(InfoExtractor):
'url': video_url, 'url': video_url,
'ext': 'mp4', 'ext': 'mp4',
'format_id': rate_str, 'format_id': rate_str,
'rate': int(rate_str), 'tbr': int(rate_str),
}) })
formats.sort(key=operator.itemgetter('rate')) self._sort_formats(formats)
description = self._html_search_regex(r'<b>Description:</b>(.*?)</p>', description = self._html_search_regex(r'<b>Description:</b>(.*?)</p>',
webpage, u'description', flags=re.DOTALL) webpage, 'description', flags=re.DOTALL)
return { return {
'id': video_id, 'id': video_id,

38
youtube_dl/extractor/mixcloud.py
View File

@ -1,4 +1,5 @@
import json from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
@ -10,17 +11,17 @@ from ..utils import (
class MixcloudIE(InfoExtractor): class MixcloudIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)' _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
IE_NAME = u'mixcloud' IE_NAME = 'mixcloud'
_TEST = { _TEST = {
u'url': u'http://www.mixcloud.com/dholbach/cryptkeeper/', 'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
u'file': u'dholbach-cryptkeeper.mp3', 'file': 'dholbach-cryptkeeper.mp3',
u'info_dict': { 'info_dict': {
u'title': u'Cryptkeeper', 'title': 'Cryptkeeper',
u'description': u'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.', 'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
u'uploader': u'Daniel Holbach', 'uploader': 'Daniel Holbach',
u'uploader_id': u'dholbach', 'uploader_id': 'dholbach',
u'upload_date': u'20111115', 'upload_date': '20111115',
}, },
} }
@ -42,18 +43,19 @@ class MixcloudIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
uploader = mobj.group(1) uploader = mobj.group(1)
cloudcast_name = mobj.group(2) cloudcast_name = mobj.group(2)
track_id = '-'.join((uploader, cloudcast_name)) track_id = '-'.join((uploader, cloudcast_name))
api_url = 'http://api.mixcloud.com/%s/%s/' % (uploader, cloudcast_name)
webpage = self._download_webpage(url, track_id)
json_data = self._download_webpage(api_url, track_id,
u'Downloading cloudcast info')
info = json.loads(json_data)
preview_url = self._search_regex(r'data-preview-url="(.+?)"', webpage, u'preview url') webpage = self._download_webpage(url, track_id)
song_url = preview_url.replace('/previews/', '/cloudcasts/originals/')
api_url = 'http://api.mixcloud.com/%s/%s/' % (uploader, cloudcast_name)
info = self._download_json(
api_url, track_id, 'Downloading cloudcast info')
preview_url = self._search_regex(
r'\s(?:data-preview-url|m-preview)="(.+?)"', webpage, 'preview url')
song_url = preview_url.replace('/previews/', '/c/originals/')
template_url = re.sub(r'(stream\d*)', 'stream%d', song_url) template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
final_song_url = self._get_url(template_url) final_song_url = self._get_url(template_url)
if final_song_url is None: if final_song_url is None:

66
youtube_dl/extractor/mpora.py Normal file
View File

@ -0,0 +1,66 @@
from __future__ import unicode_literals
import json
import re
from .common import InfoExtractor
from ..utils import (
int_or_none,
)
class MporaIE(InfoExtractor):
_VALID_URL = r'^https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
IE_NAME = 'MPORA'
_TEST = {
'url': 'http://mpora.de/videos/AAdo8okx4wiz/embed?locale=de',
'file': 'AAdo8okx4wiz.mp4',
'md5': 'a7a228473eedd3be741397cf452932eb',
'info_dict': {
'title': 'Katy Curd - Winter in the Forest',
'duration': 416,
'uploader': 'petenewman',
},
}
def _real_extract(self, url):
m = re.match(self._VALID_URL, url)
video_id = m.group('id')
webpage = self._download_webpage(url, video_id)
data_json = self._search_regex(
r"new FM\.Player\('[^']+',\s*(\{.*?)\);\n", webpage, 'json')
data = json.loads(data_json)
uploader = data['info_overlay'].get('username')
duration = data['video']['duration'] // 1000
thumbnail = data['video']['encodings']['sd']['poster']
title = data['info_overlay']['title']
formats = []
for encoding_id, edata in data['video']['encodings'].items():
for src in edata['sources']:
width_str = self._search_regex(
r'_([0-9]+)\.[a-zA-Z0-9]+$', src['src'],
False, default=None)
vcodec = src['type'].partition('/')[2]
formats.append({
'format_id': encoding_id + '-' + vcodec,
'url': src['src'],
'vcodec': vcodec,
'width': int_or_none(width_str),
})
self._sort_formats(formats)
return {
'id': video_id,
'title': title,
'formats': formats,
'uploader': uploader,
'duration': duration,
'thumbnail': thumbnail,
}

2
youtube_dl/extractor/mtv.py
View File

@ -129,7 +129,7 @@ class MTVIE(MTVServicesInfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('videoid') video_id = mobj.group('videoid')
uri = mobj.group('mgid') uri = mobj.groupdict().get('mgid')
if uri is None: if uri is None:
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)

62
youtube_dl/extractor/novamov.py Normal file
View File

@ -0,0 +1,62 @@
from __future__ import unicode_literals
import re
from .common import InfoExtractor
from ..utils import (
ExtractorError,
compat_urlparse
)
class NovamovIE(InfoExtractor):
_VALID_URL = r'http://(?:(?:www\.)?novamov\.com/video/|(?:(?:embed|www)\.)novamov\.com/embed\.php\?v=)(?P<videoid>[a-z\d]{13})'
_TEST = {
'url': 'http://www.novamov.com/video/4rurhn9x446jj',
'file': '4rurhn9x446jj.flv',
'md5': '7205f346a52bbeba427603ba10d4b935',
'info_dict': {
'title': 'search engine optimization',
'description': 'search engine optimization is used to rank the web page in the google search engine'
}
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('videoid')
page = self._download_webpage('http://www.novamov.com/video/%s' % video_id,
video_id, 'Downloading video page')
if re.search(r'This file no longer exists on our servers!</h2>', page) is not None:
raise ExtractorError(u'Video %s does not exist' % video_id, expected=True)
filekey = self._search_regex(
r'flashvars\.filekey="(?P<filekey>[^"]+)";', page, 'filekey')
title = self._html_search_regex(
r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>([^<]+)</h3>',
page, 'title', fatal=False)
description = self._html_search_regex(
r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>[^<]+</h3><p>([^<]+)</p>',
page, 'description', fatal=False)
api_response = self._download_webpage(
'http://www.novamov.com/api/player.api.php?key=%s&file=%s' % (filekey, video_id),
video_id, 'Downloading video api response')
response = compat_urlparse.parse_qs(api_response)
if 'error_msg' in response:
raise ExtractorError('novamov returned error: %s' % response['error_msg'][0], expected=True)
video_url = response['url'][0]
return {
'id': video_id,
'url': video_url,
'title': title,
'description': description
}

2
youtube_dl/extractor/nowvideo.py
View File

@ -5,7 +5,7 @@ from ..utils import compat_urlparse
class NowVideoIE(InfoExtractor): class NowVideoIE(InfoExtractor):
_VALID_URL = r'(?:https?://)?(?:www\.)?nowvideo\.ch/video/(?P<id>\w+)' _VALID_URL = r'(?:https?://)?(?:www\.)?nowvideo\.(?:ch|sx)/video/(?P<id>\w+)'
_TEST = { _TEST = {
u'url': u'http://www.nowvideo.ch/video/0mw0yow7b6dxa', u'url': u'http://www.nowvideo.ch/video/0mw0yow7b6dxa',
u'file': u'0mw0yow7b6dxa.flv', u'file': u'0mw0yow7b6dxa.flv',

116
youtube_dl/extractor/orf.py
View File

@ -1,54 +1,98 @@
# coding: utf-8 # coding: utf-8
from __future__ import unicode_literals
import re
import xml.etree.ElementTree
import json import json
import re
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
compat_urlparse, HEADRequest,
ExtractorError, unified_strdate,
find_xpath_attr,
) )
class ORFIE(InfoExtractor): class ORFIE(InfoExtractor):
_VALID_URL = r'https?://tvthek\.orf\.at/(programs/.+?/episodes|topics/.+?)/(?P<id>\d+)' _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics/.+?|program/[^/]+)/(?P<id>\d+)'
_TEST = {
'url': 'http://tvthek.orf.at/program/matinee-Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7317210/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319746/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319747',
'file': '7319747.mp4',
'md5': 'bd803c5d8c32d3c64a0ea4b4eeddf375',
'info_dict': {
'title': 'Was Sie schon immer über Klassik wissen wollten',
'description': 'md5:0ddf0d5f0060bd53f744edaa5c2e04a4',
'duration': 3508,
'upload_date': '20140105',
},
'skip': 'Blocked outside of Austria',
}
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
playlist_id = mobj.group('id') playlist_id = mobj.group('id')
webpage = self._download_webpage(url, playlist_id) webpage = self._download_webpage(url, playlist_id)
flash_xml = self._search_regex('ORF.flashXML = \'(.+?)\'', webpage, u'flash xml') data_json = self._search_regex(
flash_xml = compat_urlparse.parse_qs('xml='+flash_xml)['xml'][0] r'initializeAdworx\((.+?)\);\n', webpage, 'video info')
flash_config = xml.etree.ElementTree.fromstring(flash_xml.encode('utf-8')) all_data = json.loads(data_json)
playlist_json = self._search_regex(r'playlist\': \'(\[.*?\])\'', webpage, u'playlist').replace(r'\"','"') sdata = all_data[0]['values']['segments']
playlist = json.loads(playlist_json)
videos = [] def quality_to_int(s):
ns = '{http://tempuri.org/XMLSchema.xsd}' m = re.search('([0-9]+)', s)
xpath = '%(ns)sPlaylist/%(ns)sItems/%(ns)sItem' % {'ns': ns} if m is None:
webpage_description = self._og_search_description(webpage) return -1
for (i, (item, info)) in enumerate(zip(flash_config.findall(xpath), playlist), 1): return int(m.group(1))
# Get best quality url
rtmp_url = None entries = []
for q in ['Q6A', 'Q4A', 'Q1A']: for sd in sdata:
video_url = find_xpath_attr(item, '%sVideoUrl' % ns, 'quality', q) video_id = sd['id']
if video_url is not None: formats = [{
rtmp_url = video_url.text 'preference': -10 if fd['delivery'] == 'hls' else None,
break 'format_id': '%s-%s-%s' % (
if rtmp_url is None: fd['delivery'], fd['quality'], fd['quality_string']),
raise ExtractorError(u'Couldn\'t get video url: %s' % info['id']) 'url': fd['src'],
description = self._html_search_regex( 'protocol': fd['protocol'],
r'id="playlist_entry_%s".*?<p>(.*?)</p>' % i, webpage, 'quality': quality_to_int(fd['quality']),
u'description', default=webpage_description, flags=re.DOTALL) } for fd in sd['playlist_item_array']['sources']]
videos.append({
# Check for geoblocking.
# There is a property is_geoprotection, but that's always false
geo_str = sd.get('geoprotection_string')
if geo_str:
try:
http_url = next(
f['url']
for f in formats
if re.match(r'^https?://.*\.mp4$', f['url']))
except StopIteration:
pass
else:
req = HEADRequest(http_url)
self._request_webpage(
req, video_id,
note='Testing for geoblocking',
errnote=((
'This video seems to be blocked outside of %s. '
'You may want to try the streaming-* formats.')
% geo_str),
fatal=False)
self._sort_formats(formats)
upload_date = unified_strdate(sd['created_date'])
entries.append({
'_type': 'video', '_type': 'video',
'id': info['id'], 'id': video_id,
'title': info['title'], 'title': sd['header'],
'url': rtmp_url, 'formats': formats,
'ext': 'flv', 'description': sd.get('description'),
'description': description, 'duration': int(sd['duration_in_seconds']),
}) 'upload_date': upload_date,
'thumbnail': sd.get('image_full_url'),
})
return videos return {
'_type': 'playlist',
'entries': entries,
'id': playlist_id,
}

2
youtube_dl/extractor/pornhd.py
View File

@ -5,7 +5,7 @@ from ..utils import compat_urllib_parse
class PornHdIE(InfoExtractor): class PornHdIE(InfoExtractor):
_VALID_URL = r'(?:http://)?(?:www\.)?pornhd\.com/videos/(?P<video_id>[0-9]+)/(?P<video_title>.+)' _VALID_URL = r'(?:http://)?(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<video_id>[0-9]+)/(?P<video_title>.+)'
_TEST = { _TEST = {
u'url': u'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video', u'url': u'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
u'file': u'1962.flv', u'file': u'1962.flv',

38
youtube_dl/extractor/pornhub.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import os import os
import re import re
@ -11,16 +13,17 @@ from ..aes import (
aes_decrypt_text aes_decrypt_text
) )
class PornHubIE(InfoExtractor): class PornHubIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>pornhub\.com/view_video\.php\?viewkey=(?P<videoid>[0-9a-f]+))' _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>pornhub\.com/view_video\.php\?viewkey=(?P<videoid>[0-9a-f]+))'
_TEST = { _TEST = {
u'url': u'http://www.pornhub.com/view_video.php?viewkey=648719015', 'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
u'file': u'648719015.mp4', 'file': '648719015.mp4',
u'md5': u'882f488fa1f0026f023f33576004a2ed', 'md5': '882f488fa1f0026f023f33576004a2ed',
u'info_dict': { 'info_dict': {
u"uploader": u"BABES-COM", "uploader": "BABES-COM",
u"title": u"Seductive Indian beauty strips down and fingers her pink pussy", "title": "Seductive Indian beauty strips down and fingers her pink pussy",
u"age_limit": 18 "age_limit": 18
} }
} }
@ -33,15 +36,15 @@ class PornHubIE(InfoExtractor):
req.add_header('Cookie', 'age_verified=1') req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id) webpage = self._download_webpage(req, video_id)
video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, u'title') video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
video_uploader = self._html_search_regex(r'<b>From: </b>(?:\s|<[^>]*>)*(.+?)<', webpage, u'uploader', fatal=False) video_uploader = self._html_search_regex(r'<b>From: </b>(?:\s|<[^>]*>)*(.+?)<', webpage, 'uploader', fatal=False)
thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, u'thumbnail', fatal=False) thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
if thumbnail: if thumbnail:
thumbnail = compat_urllib_parse.unquote(thumbnail) thumbnail = compat_urllib_parse.unquote(thumbnail)
video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage))) video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
if webpage.find('"encrypted":true') != -1: if webpage.find('"encrypted":true') != -1:
password = self._html_search_regex(r'"video_title":"([^"]+)', webpage, u'password').replace('+', ' ') password = self._html_search_regex(r'"video_title":"([^"]+)', webpage, 'password').replace('+', ' ')
video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls)) video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
formats = [] formats = []
@ -50,13 +53,24 @@ class PornHubIE(InfoExtractor):
extension = os.path.splitext(path)[1][1:] extension = os.path.splitext(path)[1][1:]
format = path.split('/')[5].split('_')[:2] format = path.split('/')[5].split('_')[:2]
format = "-".join(format) format = "-".join(format)
m = re.match(r'^(?P<height>[0-9]+)P-(?P<tbr>[0-9]+)K$', format)
if m is None:
height = None
tbr = None
else:
height = int(m.group('height'))
tbr = int(m.group('tbr'))
formats.append({ formats.append({
'url': video_url, 'url': video_url,
'ext': extension, 'ext': extension,
'format': format, 'format': format,
'format_id': format, 'format_id': format,
'tbr': tbr,
'height': height,
}) })
formats.sort(key=lambda format: list(map(lambda s: s.zfill(6), format['format'].split('-')))) self._sort_formats(formats)
return { return {
'id': video_id, 'id': video_id,

2
youtube_dl/extractor/redtube.py
View File

@ -4,7 +4,7 @@ from .common import InfoExtractor
class RedTubeIE(InfoExtractor): class RedTubeIE(InfoExtractor):
_VALID_URL = r'(?:http://)?(?:www\.)?redtube\.com/(?P<id>[0-9]+)' _VALID_URL = r'http://(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
_TEST = { _TEST = {
u'url': u'http://www.redtube.com/66418', u'url': u'http://www.redtube.com/66418',
u'file': u'66418.mp4', u'file': u'66418.mp4',

2
youtube_dl/extractor/rtlnow.py
View File

@ -39,7 +39,7 @@ class RTLnowIE(InfoExtractor):
u'skip': u'Only works from Germany', u'skip': u'Only works from Germany',
}, },
{ {
u'url': u'www.voxnow.de/voxtours/suedafrika-reporter-ii.php?film_id=13883&player=1&season=17', u'url': u'http://www.voxnow.de/voxtours/suedafrika-reporter-ii.php?film_id=13883&player=1&season=17',
u'file': u'13883.flv', u'file': u'13883.flv',
u'info_dict': { u'info_dict': {
u'upload_date': u'20090627', u'upload_date': u'20090627',

107
youtube_dl/extractor/soundcloud.py
View File

@ -1,4 +1,6 @@
# encoding: utf-8 # encoding: utf-8
from __future__ import unicode_literals
import json import json
import re import re
import itertools import itertools
@ -29,61 +31,61 @@ class SoundcloudIE(InfoExtractor):
(?!sets/)(?P<title>[\w\d-]+)/? (?!sets/)(?P<title>[\w\d-]+)/?
(?P<token>[^?]+?)?(?:[?].*)?$) (?P<token>[^?]+?)?(?:[?].*)?$)
|(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)) |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+))
|(?P<widget>w\.soundcloud\.com/player/?.*?url=.*) |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
) )
''' '''
IE_NAME = u'soundcloud' IE_NAME = 'soundcloud'
_TESTS = [ _TESTS = [
{ {
u'url': u'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy', 'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
u'file': u'62986583.mp3', 'file': '62986583.mp3',
u'md5': u'ebef0a451b909710ed1d7787dddbf0d7', 'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
u'info_dict': { 'info_dict': {
u"upload_date": u"20121011", "upload_date": "20121011",
u"description": u"No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd", "description": "No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd",
u"uploader": u"E.T. ExTerrestrial Music", "uploader": "E.T. ExTerrestrial Music",
u"title": u"Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1" "title": "Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1"
} }
}, },
# not streamable song # not streamable song
{ {
u'url': u'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep', 'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
u'info_dict': { 'info_dict': {
u'id': u'47127627', 'id': '47127627',
u'ext': u'mp3', 'ext': 'mp3',
u'title': u'Goldrushed', 'title': 'Goldrushed',
u'uploader': u'The Royal Concept', 'uploader': 'The Royal Concept',
u'upload_date': u'20120521', 'upload_date': '20120521',
}, },
u'params': { 'params': {
# rtmp # rtmp
u'skip_download': True, 'skip_download': True,
}, },
}, },
# private link # private link
{ {
u'url': u'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp', 'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp',
u'md5': u'aa0dd32bfea9b0c5ef4f02aacd080604', 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
u'info_dict': { 'info_dict': {
u'id': u'123998367', 'id': '123998367',
u'ext': u'mp3', 'ext': 'mp3',
u'title': u'Youtube - Dl Test Video \'\' Ä↭', 'title': 'Youtube - Dl Test Video \'\' Ä↭',
u'uploader': u'jaimeMF', 'uploader': 'jaimeMF',
u'description': u'test chars: \"\'/\\ä↭', 'description': 'test chars: \"\'/\\ä↭',
u'upload_date': u'20131209', 'upload_date': '20131209',
}, },
}, },
# downloadable song # downloadable song
{ {
u'url': u'https://soundcloud.com/simgretina/just-your-problem-baby-1', 'url': 'https://soundcloud.com/simgretina/just-your-problem-baby-1',
u'md5': u'56a8b69568acaa967b4c49f9d1d52d19', 'md5': '56a8b69568acaa967b4c49f9d1d52d19',
u'info_dict': { 'info_dict': {
u'id': u'105614606', 'id': '105614606',
u'ext': u'wav', 'ext': 'wav',
u'title': u'Just Your Problem Baby (Acapella)', 'title': 'Just Your Problem Baby (Acapella)',
u'description': u'Vocals', 'description': 'Vocals',
u'uploader': u'Sim Gretina', 'uploader': 'Sim Gretina',
u'upload_date': u'20130815', 'upload_date': '20130815',
}, },
}, },
] ]
@ -112,7 +114,7 @@ class SoundcloudIE(InfoExtractor):
thumbnail = info['artwork_url'] thumbnail = info['artwork_url']
if thumbnail is not None: if thumbnail is not None:
thumbnail = thumbnail.replace('-large', '-t500x500') thumbnail = thumbnail.replace('-large', '-t500x500')
ext = u'mp3' ext = 'mp3'
result = { result = {
'id': track_id, 'id': track_id,
'uploader': info['user']['username'], 'uploader': info['user']['username'],
@ -124,11 +126,11 @@ class SoundcloudIE(InfoExtractor):
if info.get('downloadable', False): if info.get('downloadable', False):
# We can build a direct link to the song # We can build a direct link to the song
format_url = ( format_url = (
u'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format( 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(
track_id, self._CLIENT_ID)) track_id, self._CLIENT_ID))
result['formats'] = [{ result['formats'] = [{
'format_id': 'download', 'format_id': 'download',
'ext': info.get('original_format', u'mp3'), 'ext': info.get('original_format', 'mp3'),
'url': format_url, 'url': format_url,
'vcodec': 'none', 'vcodec': 'none',
}] }]
@ -138,7 +140,7 @@ class SoundcloudIE(InfoExtractor):
'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token)) 'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token))
stream_json = self._download_webpage( stream_json = self._download_webpage(
streams_url, streams_url,
track_id, u'Downloading track url') track_id, 'Downloading track url')
formats = [] formats = []
format_dict = json.loads(stream_json) format_dict = json.loads(stream_json)
@ -165,20 +167,19 @@ class SoundcloudIE(InfoExtractor):
# We fallback to the stream_url in the original info, this # We fallback to the stream_url in the original info, this
# cannot be always used, sometimes it can give an HTTP 404 error # cannot be always used, sometimes it can give an HTTP 404 error
formats.append({ formats.append({
'format_id': u'fallback', 'format_id': 'fallback',
'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID, 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID,
'ext': ext, 'ext': ext,
'vcodec': 'none', 'vcodec': 'none',
}) })
def format_pref(f): for f in formats:
if f['format_id'].startswith('http'): if f['format_id'].startswith('http'):
return 2 f['protocol'] = 'http'
if f['format_id'].startswith('rtmp'): if f['format_id'].startswith('rtmp'):
return 1 f['protocol'] = 'rtmp'
return 0
formats.sort(key=format_pref) self._sort_formats(formats)
result['formats'] = formats result['formats'] = formats
return result return result
@ -193,7 +194,7 @@ class SoundcloudIE(InfoExtractor):
if track_id is not None: if track_id is not None:
info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
full_title = track_id full_title = track_id
elif mobj.group('widget'): elif mobj.group('player'):
query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query) query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
return self.url_result(query['url'][0], ie='Soundcloud') return self.url_result(query['url'][0], ie='Soundcloud')
else: else:
@ -210,14 +211,14 @@ class SoundcloudIE(InfoExtractor):
url = 'http://soundcloud.com/%s' % resolve_title url = 'http://soundcloud.com/%s' % resolve_title
info_json_url = self._resolv_url(url) info_json_url = self._resolv_url(url)
info_json = self._download_webpage(info_json_url, full_title, u'Downloading info JSON') info_json = self._download_webpage(info_json_url, full_title, 'Downloading info JSON')
info = json.loads(info_json) info = json.loads(info_json)
return self._extract_info_dict(info, full_title, secret_token=token) return self._extract_info_dict(info, full_title, secret_token=token)
class SoundcloudSetIE(SoundcloudIE): class SoundcloudSetIE(SoundcloudIE):
_VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)(?:[?].*)?$' _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)(?:[?].*)?$'
IE_NAME = u'soundcloud:set' IE_NAME = 'soundcloud:set'
# it's in tests/test_playlists.py # it's in tests/test_playlists.py
_TESTS = [] _TESTS = []
@ -254,7 +255,7 @@ class SoundcloudSetIE(SoundcloudIE):
class SoundcloudUserIE(SoundcloudIE): class SoundcloudUserIE(SoundcloudIE):
_VALID_URL = r'https?://(www\.)?soundcloud\.com/(?P<user>[^/]+)(/?(tracks/)?)?(\?.*)?$' _VALID_URL = r'https?://(www\.)?soundcloud\.com/(?P<user>[^/]+)(/?(tracks/)?)?(\?.*)?$'
IE_NAME = u'soundcloud:user' IE_NAME = 'soundcloud:user'
# it's in tests/test_playlists.py # it's in tests/test_playlists.py
_TESTS = [] _TESTS = []
@ -266,7 +267,7 @@ class SoundcloudUserIE(SoundcloudIE):
url = 'http://soundcloud.com/%s/' % uploader url = 'http://soundcloud.com/%s/' % uploader
resolv_url = self._resolv_url(url) resolv_url = self._resolv_url(url)
user_json = self._download_webpage(resolv_url, uploader, user_json = self._download_webpage(resolv_url, uploader,
u'Downloading user info') 'Downloading user info')
user = json.loads(user_json) user = json.loads(user_json)
tracks = [] tracks = []
@ -276,7 +277,7 @@ class SoundcloudUserIE(SoundcloudIE):
}) })
tracks_url = 'http://api.soundcloud.com/users/%s/tracks.json?' % user['id'] + data tracks_url = 'http://api.soundcloud.com/users/%s/tracks.json?' % user['id'] + data
response = self._download_webpage(tracks_url, uploader, response = self._download_webpage(tracks_url, uploader,
u'Downloading tracks page %s' % (i+1)) 'Downloading tracks page %s' % (i+1))
new_tracks = json.loads(response) new_tracks = json.loads(response)
tracks.extend(self._extract_info_dict(track, quiet=True) for track in new_tracks) tracks.extend(self._extract_info_dict(track, quiet=True) for track in new_tracks)
if len(new_tracks) < 50: if len(new_tracks) < 50:

38
youtube_dl/extractor/spankwire.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import os import os
import re import re
@ -11,17 +13,18 @@ from ..aes import (
aes_decrypt_text aes_decrypt_text
) )
class SpankwireIE(InfoExtractor): class SpankwireIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<videoid>[0-9]+)/?)' _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<videoid>[0-9]+)/?)'
_TEST = { _TEST = {
u'url': u'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/', 'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
u'file': u'103545.mp4', 'file': '103545.mp4',
u'md5': u'1b3f55e345500552dbc252a3e9c1af43', 'md5': '1b3f55e345500552dbc252a3e9c1af43',
u'info_dict': { 'info_dict': {
u"uploader": u"oreusz", "uploader": "oreusz",
u"title": u"Buckcherry`s X Rated Music Video Crazy Bitch", "title": "Buckcherry`s X Rated Music Video Crazy Bitch",
u"description": u"Crazy Bitch X rated music video.", "description": "Crazy Bitch X rated music video.",
u"age_limit": 18, "age_limit": 18,
} }
} }
@ -34,17 +37,17 @@ class SpankwireIE(InfoExtractor):
req.add_header('Cookie', 'age_verified=1') req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id) webpage = self._download_webpage(req, video_id)
video_title = self._html_search_regex(r'<h1>([^<]+)', webpage, u'title') video_title = self._html_search_regex(r'<h1>([^<]+)', webpage, 'title')
video_uploader = self._html_search_regex( video_uploader = self._html_search_regex(
r'by:\s*<a [^>]*>(.+?)</a>', webpage, u'uploader', fatal=False) r'by:\s*<a [^>]*>(.+?)</a>', webpage, 'uploader', fatal=False)
thumbnail = self._html_search_regex( thumbnail = self._html_search_regex(
r'flashvars\.image_url = "([^"]+)', webpage, u'thumbnail', fatal=False) r'flashvars\.image_url = "([^"]+)', webpage, 'thumbnail', fatal=False)
description = self._html_search_regex( description = self._html_search_regex(
r'<div\s+id="descriptionContent">([^<]+)<', webpage, u'description', fatal=False) r'<div\s+id="descriptionContent">([^<]+)<', webpage, 'description', fatal=False)
video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'flashvars\.quality_[0-9]{3}p = "([^"]+)', webpage))) video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'flashvars\.quality_[0-9]{3}p = "([^"]+)', webpage)))
if webpage.find('flashvars\.encrypted = "true"') != -1: if webpage.find('flashvars\.encrypted = "true"') != -1:
password = self._html_search_regex(r'flashvars\.video_title = "([^"]+)', webpage, u'password').replace('+', ' ') password = self._html_search_regex(r'flashvars\.video_title = "([^"]+)', webpage, 'password').replace('+', ' ')
video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls)) video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
formats = [] formats = []
@ -52,14 +55,21 @@ class SpankwireIE(InfoExtractor):
path = compat_urllib_parse_urlparse(video_url).path path = compat_urllib_parse_urlparse(video_url).path
extension = os.path.splitext(path)[1][1:] extension = os.path.splitext(path)[1][1:]
format = path.split('/')[4].split('_')[:2] format = path.split('/')[4].split('_')[:2]
resolution, bitrate_str = format
format = "-".join(format) format = "-".join(format)
height = int(resolution.rstrip('P'))
tbr = int(bitrate_str.rstrip('K'))
formats.append({ formats.append({
'url': video_url, 'url': video_url,
'ext': extension, 'ext': extension,
'resolution': resolution,
'format': format, 'format': format,
'tbr': tbr,
'height': height,
'format_id': format, 'format_id': format,
}) })
formats.sort(key=lambda format: list(map(lambda s: s.zfill(6), format['format'].split('-')))) self._sort_formats(formats)
age_limit = self._rta_search(webpage) age_limit = self._rta_search(webpage)

57
youtube_dl/extractor/teamcoco.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
@ -9,61 +11,66 @@ from ..utils import (
class TeamcocoIE(InfoExtractor): class TeamcocoIE(InfoExtractor):
_VALID_URL = r'http://teamcoco\.com/video/(?P<url_title>.*)' _VALID_URL = r'http://teamcoco\.com/video/(?P<url_title>.*)'
_TEST = { _TEST = {
u'url': u'http://teamcoco.com/video/louis-ck-interview-george-w-bush', 'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
u'file': u'19705.mp4', 'file': '19705.mp4',
u'md5': u'cde9ba0fa3506f5f017ce11ead928f9a', 'md5': 'cde9ba0fa3506f5f017ce11ead928f9a',
u'info_dict': { 'info_dict': {
u"description": u"Louis C.K. got starstruck by George W. Bush, so what? Part one.", "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
u"title": u"Louis C.K. Interview Pt. 1 11/3/11" "title": "Louis C.K. Interview Pt. 1 11/3/11"
} }
} }
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
if mobj is None: if mobj is None:
raise ExtractorError(u'Invalid URL: %s' % url) raise ExtractorError('Invalid URL: %s' % url)
url_title = mobj.group('url_title') url_title = mobj.group('url_title')
webpage = self._download_webpage(url, url_title) webpage = self._download_webpage(url, url_title)
video_id = self._html_search_regex(r'<article class="video" data-id="(\d+?)"', video_id = self._html_search_regex(
webpage, u'video id') r'<article class="video" data-id="(\d+?)"',
webpage, 'video id')
self.report_extraction(video_id) self.report_extraction(video_id)
data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
data = self._download_xml(data_url, video_id, 'Downloading data webpage') data = self._download_xml(data_url, video_id, 'Downloading data webpage')
qualities = ['500k', '480p', '1000k', '720p', '1080p'] qualities = ['500k', '480p', '1000k', '720p', '1080p']
formats = [] formats = []
for file in data.findall('files/file'): for filed in data.findall('files/file'):
if file.attrib.get('playmode') == 'all': if filed.attrib.get('playmode') == 'all':
# it just duplicates one of the entries # it just duplicates one of the entries
break break
file_url = file.text file_url = filed.text
m_format = re.search(r'(\d+(k|p))\.mp4', file_url) m_format = re.search(r'(\d+(k|p))\.mp4', file_url)
if m_format is not None: if m_format is not None:
format_id = m_format.group(1) format_id = m_format.group(1)
else: else:
format_id = file.attrib['bitrate'] format_id = filed.attrib['bitrate']
tbr = (
int(filed.attrib['bitrate'])
if filed.attrib['bitrate'].isdigit()
else None)
try:
quality = qualities.index(format_id)
except ValueError:
quality = -1
formats.append({ formats.append({
'url': file_url, 'url': file_url,
'ext': 'mp4', 'ext': 'mp4',
'tbr': tbr,
'format_id': format_id, 'format_id': format_id,
'quality': quality,
}) })
def sort_key(f):
try: self._sort_formats(formats)
return qualities.index(f['format_id'])
except ValueError:
return -1
formats.sort(key=sort_key)
if not formats:
raise ExtractorError(u'Unable to extract video URL')
return { return {
'id': video_id, 'id': video_id,
'formats': formats, 'formats': formats,
'title': self._og_search_title(webpage), 'title': self._og_search_title(webpage),
'thumbnail': self._og_search_thumbnail(webpage), 'thumbnail': self._og_search_thumbnail(webpage),
'description': self._og_search_description(webpage), 'description': self._og_search_description(webpage),
} }

17
youtube_dl/extractor/ted.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import json import json
import re import re
@ -7,6 +9,7 @@ from ..utils import (
RegexNotFoundError, RegexNotFoundError,
) )
class TEDIE(SubtitlesInfoExtractor): class TEDIE(SubtitlesInfoExtractor):
_VALID_URL=r'''http://www\.ted\.com/ _VALID_URL=r'''http://www\.ted\.com/
( (
@ -18,12 +21,12 @@ class TEDIE(SubtitlesInfoExtractor):
/(?P<name>\w+) # Here goes the name and then ".html" /(?P<name>\w+) # Here goes the name and then ".html"
''' '''
_TEST = { _TEST = {
u'url': u'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html', 'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
u'file': u'102.mp4', 'file': '102.mp4',
u'md5': u'2d76ee1576672e0bd8f187513267adf6', 'md5': '4ea1dada91e4174b53dac2bb8ace429d',
u'info_dict': { 'info_dict': {
u"description": u"md5:c6fa72e6eedbd938c9caf6b2702f5922", "description": "md5:c6fa72e6eedbd938c9caf6b2702f5922",
u"title": u"Dan Dennett: The illusion of consciousness" "title": "Dan Dennett: The illusion of consciousness"
} }
} }
@ -47,7 +50,7 @@ class TEDIE(SubtitlesInfoExtractor):
'''Returns the videos of the playlist''' '''Returns the videos of the playlist'''
webpage = self._download_webpage( webpage = self._download_webpage(
url, playlist_id, u'Downloading playlist webpage') url, playlist_id, 'Downloading playlist webpage')
matches = re.finditer( matches = re.finditer(
r'<p\s+class="talk-title[^"]*"><a\s+href="(?P<talk_url>/talks/[^"]+\.html)">[^<]*</a></p>', r'<p\s+class="talk-title[^"]*"><a\s+href="(?P<talk_url>/talks/[^"]+\.html)">[^<]*</a></p>',
webpage) webpage)

14
youtube_dl/extractor/theplatform.py
View File

@ -55,15 +55,21 @@ class ThePlatformIE(InfoExtractor):
formats = [] formats = []
for f in switch.findall(_x('smil:video')): for f in switch.findall(_x('smil:video')):
attr = f.attrib attr = f.attrib
width = int(attr['width'])
height = int(attr['height'])
vbr = int(attr['system-bitrate']) // 1000
format_id = '%dx%d_%dk' % (width, height, vbr)
formats.append({ formats.append({
'format_id': format_id,
'url': base_url, 'url': base_url,
'play_path': 'mp4:' + attr['src'], 'play_path': 'mp4:' + attr['src'],
'ext': 'flv', 'ext': 'flv',
'width': int(attr['width']), 'width': width,
'height': int(attr['height']), 'height': height,
'vbr': int(attr['system-bitrate']), 'vbr': vbr,
}) })
formats.sort(key=lambda f: (f['height'], f['width'], f['vbr']))
self._sort_formats(formats)
return { return {
'id': video_id, 'id': video_id,

39
youtube_dl/extractor/veehd.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import re import re
import json import json
@ -8,16 +10,17 @@ from ..utils import (
clean_html, clean_html,
) )
class VeeHDIE(InfoExtractor): class VeeHDIE(InfoExtractor):
_VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)' _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)'
_TEST = { _TEST = {
u'url': u'http://veehd.com/video/4686958', 'url': 'http://veehd.com/video/4686958',
u'file': u'4686958.mp4', 'file': '4686958.mp4',
u'info_dict': { 'info_dict': {
u'title': u'Time Lapse View from Space ( ISS)', 'title': 'Time Lapse View from Space ( ISS)',
u'uploader_id': u'spotted', 'uploader_id': 'spotted',
u'description': u'md5:f0094c4cf3a72e22bc4e4239ef767ad7', 'description': 'md5:f0094c4cf3a72e22bc4e4239ef767ad7',
}, },
} }
@ -25,24 +28,30 @@ class VeeHDIE(InfoExtractor):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id') video_id = mobj.group('id')
# VeeHD seems to send garbage on the first request.
# See https://github.com/rg3/youtube-dl/issues/2102
self._download_webpage(url, video_id, 'Requesting webpage')
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
player_path = self._search_regex(r'\$\("#playeriframe"\).attr\({src : "(.+?)"', player_path = self._search_regex(
webpage, u'player path') r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
webpage, 'player path')
player_url = compat_urlparse.urljoin(url, player_path) player_url = compat_urlparse.urljoin(url, player_path)
player_page = self._download_webpage(player_url, video_id,
u'Downloading player page') self._download_webpage(player_url, video_id, 'Requesting player page')
config_json = self._search_regex(r'value=\'config=({.+?})\'', player_page = self._download_webpage(
player_page, u'config json') player_url, video_id, 'Downloading player page')
config_json = self._search_regex(
r'value=\'config=({.+?})\'', player_page, 'config json')
config = json.loads(config_json) config = json.loads(config_json)
video_url = compat_urlparse.unquote(config['clip']['url']) video_url = compat_urlparse.unquote(config['clip']['url'])
title = clean_html(get_element_by_id('videoName', webpage).rpartition('|')[0]) title = clean_html(get_element_by_id('videoName', webpage).rpartition('|')[0])
uploader_id = self._html_search_regex(r'<a href="/profile/\d+">(.+?)</a>', uploader_id = self._html_search_regex(r'<a href="/profile/\d+">(.+?)</a>',
webpage, u'uploader') webpage, 'uploader')
thumbnail = self._search_regex(r'<img id="veehdpreview" src="(.+?)"', thumbnail = self._search_regex(r'<img id="veehdpreview" src="(.+?)"',
webpage, u'thumbnail') webpage, 'thumbnail')
description = self._html_search_regex(r'<td class="infodropdown".*?<div>(.*?)<ul', description = self._html_search_regex(r'<td class="infodropdown".*?<div>(.*?)<ul',
webpage, u'description', flags=re.DOTALL) webpage, 'description', flags=re.DOTALL)
return { return {
'_type': 'video', '_type': 'video',

44
youtube_dl/extractor/veoh.py
View File

@ -1,22 +1,22 @@
from __future__ import unicode_literals
import re import re
import json import json
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import (
determine_ext,
)
class VeohIE(InfoExtractor): class VeohIE(InfoExtractor):
_VALID_URL = r'http://www\.veoh\.com/watch/v(?P<id>\d*)' _VALID_URL = r'http://(?:www\.)?veoh\.com/(?:watch|iphone/#_Watch)/v(?P<id>\d*)'
_TEST = { _TEST = {
u'url': u'http://www.veoh.com/watch/v56314296nk7Zdmz3', 'url': 'http://www.veoh.com/watch/v56314296nk7Zdmz3',
u'file': u'56314296.mp4', 'file': '56314296.mp4',
u'md5': u'620e68e6a3cff80086df3348426c9ca3', 'md5': '620e68e6a3cff80086df3348426c9ca3',
u'info_dict': { 'info_dict': {
u'title': u'Straight Backs Are Stronger', 'title': 'Straight Backs Are Stronger',
u'uploader': u'LUMOback', 'uploader': 'LUMOback',
u'description': u'At LUMOback, we believe straight backs are stronger. The LUMOback Posture & Movement Sensor: It gently vibrates when you slouch, inspiring improved posture and mobility. Use the app to track your data and improve your posture over time. ', 'description': 'At LUMOback, we believe straight backs are stronger. The LUMOback Posture & Movement Sensor: It gently vibrates when you slouch, inspiring improved posture and mobility. Use the app to track your data and improve your posture over time. ',
} }
} }
@ -28,20 +28,20 @@ class VeohIE(InfoExtractor):
m_youtube = re.search(r'http://www\.youtube\.com/v/(.*?)(\&|")', webpage) m_youtube = re.search(r'http://www\.youtube\.com/v/(.*?)(\&|")', webpage)
if m_youtube is not None: if m_youtube is not None:
youtube_id = m_youtube.group(1) youtube_id = m_youtube.group(1)
self.to_screen(u'%s: detected Youtube video.' % video_id) self.to_screen('%s: detected Youtube video.' % video_id)
return self.url_result(youtube_id, 'Youtube') return self.url_result(youtube_id, 'Youtube')
self.report_extraction(video_id) self.report_extraction(video_id)
info = self._search_regex(r'videoDetailsJSON = \'({.*?})\';', webpage, 'info') info = self._search_regex(r'videoDetailsJSON = \'({.*?})\';', webpage, 'info')
info = json.loads(info) info = json.loads(info)
video_url = info.get('fullPreviewHashHighPath') or info.get('fullPreviewHashLowPath') video_url = info.get('fullPreviewHashHighPath') or info.get('fullPreviewHashLowPath')
return {'id': info['videoId'], return {
'title': info['title'], 'id': info['videoId'],
'ext': determine_ext(video_url), 'title': info['title'],
'url': video_url, 'url': video_url,
'uploader': info['username'], 'uploader': info['username'],
'thumbnail': info.get('highResImage') or info.get('medResImage'), 'thumbnail': info.get('highResImage') or info.get('medResImage'),
'description': info['description'], 'description': info['description'],
'view_count': info['views'], 'view_count': info['views'],
} }

161
youtube_dl/extractor/vimeo.py
View File

@ -1,4 +1,6 @@
# encoding: utf-8 # encoding: utf-8
from __future__ import unicode_literals
import json import json
import re import re
import itertools import itertools
@ -22,7 +24,7 @@ class VimeoIE(InfoExtractor):
# _VALID_URL matches Vimeo URLs # _VALID_URL matches Vimeo URLs
_VALID_URL = r'''(?x) _VALID_URL = r'''(?x)
(?P<proto>https?://)? (?P<proto>(?:https?:)?//)?
(?:(?:www|(?P<player>player))\.)? (?:(?:www|(?P<player>player))\.)?
vimeo(?P<pro>pro)?\.com/ vimeo(?P<pro>pro)?\.com/
(?:.*?/)? (?:.*?/)?
@ -31,54 +33,55 @@ class VimeoIE(InfoExtractor):
(?P<id>[0-9]+) (?P<id>[0-9]+)
/?(?:[?&].*)?(?:[#].*)?$''' /?(?:[?&].*)?(?:[#].*)?$'''
_NETRC_MACHINE = 'vimeo' _NETRC_MACHINE = 'vimeo'
IE_NAME = u'vimeo' IE_NAME = 'vimeo'
_TESTS = [ _TESTS = [
{ {
u'url': u'http://vimeo.com/56015672#at=0', 'url': 'http://vimeo.com/56015672#at=0',
u'file': u'56015672.mp4', 'file': '56015672.mp4',
u'md5': u'8879b6cc097e987f02484baf890129e5', 'md5': '8879b6cc097e987f02484baf890129e5',
u'info_dict': { 'info_dict': {
u"upload_date": u"20121220", "upload_date": "20121220",
u"description": u"This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550", "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
u"uploader_id": u"user7108434", "uploader_id": "user7108434",
u"uploader": u"Filippo Valsorda", "uploader": "Filippo Valsorda",
u"title": u"youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550", "title": "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
}, },
}, },
{ {
u'url': u'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876', 'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
u'file': u'68093876.mp4', 'file': '68093876.mp4',
u'md5': u'3b5ca6aa22b60dfeeadf50b72e44ed82', 'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
u'note': u'Vimeo Pro video (#1197)', 'note': 'Vimeo Pro video (#1197)',
u'info_dict': { 'info_dict': {
u'uploader_id': u'openstreetmapus', 'uploader_id': 'openstreetmapus',
u'uploader': u'OpenStreetMap US', 'uploader': 'OpenStreetMap US',
u'title': u'Andy Allan - Putting the Carto into OpenStreetMap Cartography', 'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
}, },
}, },
{ {
u'url': u'http://player.vimeo.com/video/54469442', 'url': 'http://player.vimeo.com/video/54469442',
u'file': u'54469442.mp4', 'file': '54469442.mp4',
u'md5': u'619b811a4417aa4abe78dc653becf511', 'md5': '619b811a4417aa4abe78dc653becf511',
u'note': u'Videos that embed the url in the player page', 'note': 'Videos that embed the url in the player page',
u'info_dict': { 'info_dict': {
u'title': u'Kathy Sierra: Building the minimum Badass User, Business of Software', 'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software',
u'uploader': u'The BLN & Business of Software', 'uploader': 'The BLN & Business of Software',
'uploader_id': 'theblnbusinessofsoftware',
}, },
}, },
{ {
u'url': u'http://vimeo.com/68375962', 'url': 'http://vimeo.com/68375962',
u'file': u'68375962.mp4', 'file': '68375962.mp4',
u'md5': u'aaf896bdb7ddd6476df50007a0ac0ae7', 'md5': 'aaf896bdb7ddd6476df50007a0ac0ae7',
u'note': u'Video protected with password', 'note': 'Video protected with password',
u'info_dict': { 'info_dict': {
u'title': u'youtube-dl password protected test video', 'title': 'youtube-dl password protected test video',
u'upload_date': u'20130614', 'upload_date': '20130614',
u'uploader_id': u'user18948128', 'uploader_id': 'user18948128',
u'uploader': u'Jaime Marquínez Ferrándiz', 'uploader': 'Jaime Marquínez Ferrándiz',
}, },
u'params': { 'params': {
u'videopassword': u'youtube-dl', 'videopassword': 'youtube-dl',
}, },
}, },
] ]
@ -90,7 +93,7 @@ class VimeoIE(InfoExtractor):
self.report_login() self.report_login()
login_url = 'https://vimeo.com/log_in' login_url = 'https://vimeo.com/log_in'
webpage = self._download_webpage(login_url, None, False) webpage = self._download_webpage(login_url, None, False)
token = re.search(r'xsrft: \'(.*?)\'', webpage).group(1) token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
data = compat_urllib_parse.urlencode({'email': username, data = compat_urllib_parse.urlencode({'email': username,
'password': password, 'password': password,
'action': 'login', 'action': 'login',
@ -100,13 +103,13 @@ class VimeoIE(InfoExtractor):
login_request = compat_urllib_request.Request(login_url, data) login_request = compat_urllib_request.Request(login_url, data)
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded') login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
login_request.add_header('Cookie', 'xsrft=%s' % token) login_request.add_header('Cookie', 'xsrft=%s' % token)
self._download_webpage(login_request, None, False, u'Wrong login info') self._download_webpage(login_request, None, False, 'Wrong login info')
def _verify_video_password(self, url, video_id, webpage): def _verify_video_password(self, url, video_id, webpage):
password = self._downloader.params.get('videopassword', None) password = self._downloader.params.get('videopassword', None)
if password is None: if password is None:
raise ExtractorError(u'This video is protected by a password, use the --video-password option') raise ExtractorError('This video is protected by a password, use the --video-password option')
token = re.search(r'xsrft: \'(.*?)\'', webpage).group(1) token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
data = compat_urllib_parse.urlencode({'password': password, data = compat_urllib_parse.urlencode({'password': password,
'token': token}) 'token': token})
# I didn't manage to use the password with https # I didn't manage to use the password with https
@ -118,8 +121,21 @@ class VimeoIE(InfoExtractor):
password_request.add_header('Content-Type', 'application/x-www-form-urlencoded') password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
password_request.add_header('Cookie', 'xsrft=%s' % token) password_request.add_header('Cookie', 'xsrft=%s' % token)
self._download_webpage(password_request, video_id, self._download_webpage(password_request, video_id,
u'Verifying the password', 'Verifying the password',
u'Wrong password') 'Wrong password')
def _verify_player_video_password(self, url, video_id):
password = self._downloader.params.get('videopassword', None)
if password is None:
raise ExtractorError('This video is protected by a password, use the --video-password option')
data = compat_urllib_parse.urlencode({'password': password})
pass_url = url + '/check-password'
password_request = compat_urllib_request.Request(pass_url, data)
password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
return self._download_json(
password_request, video_id,
'Verifying the password',
'Wrong password')
def _real_initialize(self): def _real_initialize(self):
self._login() self._login()
@ -133,9 +149,6 @@ class VimeoIE(InfoExtractor):
# Extract ID from URL # Extract ID from URL
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
if mobj is None:
raise ExtractorError(u'Invalid URL: %s' % url)
video_id = mobj.group('id') video_id = mobj.group('id')
if mobj.group('pro') or mobj.group('player'): if mobj.group('pro') or mobj.group('player'):
url = 'http://player.vimeo.com/video/' + video_id url = 'http://player.vimeo.com/video/' + video_id
@ -155,7 +168,7 @@ class VimeoIE(InfoExtractor):
try: try:
try: try:
config_url = self._html_search_regex( config_url = self._html_search_regex(
r' data-config-url="(.+?)"', webpage, u'config URL') r' data-config-url="(.+?)"', webpage, 'config URL')
config_json = self._download_webpage(config_url, video_id) config_json = self._download_webpage(config_url, video_id)
config = json.loads(config_json) config = json.loads(config_json)
except RegexNotFoundError: except RegexNotFoundError:
@ -166,19 +179,22 @@ class VimeoIE(InfoExtractor):
config_re = r'%s=({.+?});' % re.escape(m_variable_name.group(1)) config_re = r'%s=({.+?});' % re.escape(m_variable_name.group(1))
else: else:
config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});'] config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});']
config = self._search_regex(config_re, webpage, u'info section', config = self._search_regex(config_re, webpage, 'info section',
flags=re.DOTALL) flags=re.DOTALL)
config = json.loads(config) config = json.loads(config)
except Exception as e: except Exception as e:
if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage): if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
raise ExtractorError(u'The author has restricted the access to this video, try with the "--referer" option') raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
if re.search('<form[^>]+?id="pw_form"', webpage) is not None: if re.search('<form[^>]+?id="pw_form"', webpage) is not None:
self._verify_video_password(url, video_id, webpage) self._verify_video_password(url, video_id, webpage)
return self._real_extract(url) return self._real_extract(url)
else: else:
raise ExtractorError(u'Unable to extract info section', raise ExtractorError('Unable to extract info section',
cause=e) cause=e)
else:
if config.get('view') == 4:
config = self._verify_player_video_password(url, video_id)
# Extract title # Extract title
video_title = config["video"]["title"] video_title = config["video"]["title"]
@ -212,9 +228,9 @@ class VimeoIE(InfoExtractor):
video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3) video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
try: try:
view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, u'view count')) view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, 'view count'))
like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, u'like count')) like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, 'like count'))
comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, u'comment count')) comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, 'comment count'))
except RegexNotFoundError: except RegexNotFoundError:
# This info is only available in vimeo.com/{id} urls # This info is only available in vimeo.com/{id} urls
view_count = None view_count = None
@ -255,7 +271,7 @@ class VimeoIE(InfoExtractor):
for key in ('other', 'sd', 'hd'): for key in ('other', 'sd', 'hd'):
formats += files[key] formats += files[key]
if len(formats) == 0: if len(formats) == 0:
raise ExtractorError(u'No known codec found') raise ExtractorError('No known codec found')
return { return {
'id': video_id, 'id': video_id,
@ -274,7 +290,7 @@ class VimeoIE(InfoExtractor):
class VimeoChannelIE(InfoExtractor): class VimeoChannelIE(InfoExtractor):
IE_NAME = u'vimeo:channel' IE_NAME = 'vimeo:channel'
_VALID_URL = r'(?:https?://)?vimeo.\com/channels/(?P<id>[^/]+)' _VALID_URL = r'(?:https?://)?vimeo.\com/channels/(?P<id>[^/]+)'
_MORE_PAGES_INDICATOR = r'<a.+?rel="next"' _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
_TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"' _TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"'
@ -283,14 +299,14 @@ class VimeoChannelIE(InfoExtractor):
return '%s/videos/page:%d/' % (base_url, pagenum) return '%s/videos/page:%d/' % (base_url, pagenum)
def _extract_list_title(self, webpage): def _extract_list_title(self, webpage):
return self._html_search_regex(self._TITLE_RE, webpage, u'list title') return self._html_search_regex(self._TITLE_RE, webpage, 'list title')
def _extract_videos(self, list_id, base_url): def _extract_videos(self, list_id, base_url):
video_ids = [] video_ids = []
for pagenum in itertools.count(1): for pagenum in itertools.count(1):
webpage = self._download_webpage( webpage = self._download_webpage(
self._page_url(base_url, pagenum) ,list_id, self._page_url(base_url, pagenum) ,list_id,
u'Downloading page %s' % pagenum) 'Downloading page %s' % pagenum)
video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage)) video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None: if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
break break
@ -310,8 +326,8 @@ class VimeoChannelIE(InfoExtractor):
class VimeoUserIE(VimeoChannelIE): class VimeoUserIE(VimeoChannelIE):
IE_NAME = u'vimeo:user' IE_NAME = 'vimeo:user'
_VALID_URL = r'(?:https?://)?vimeo.\com/(?P<name>[^/]+)' _VALID_URL = r'(?:https?://)?vimeo.\com/(?P<name>[^/]+)(?:/videos|[#?]|$)'
_TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>' _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'
@classmethod @classmethod
@ -327,7 +343,7 @@ class VimeoUserIE(VimeoChannelIE):
class VimeoAlbumIE(VimeoChannelIE): class VimeoAlbumIE(VimeoChannelIE):
IE_NAME = u'vimeo:album' IE_NAME = 'vimeo:album'
_VALID_URL = r'(?:https?://)?vimeo.\com/album/(?P<id>\d+)' _VALID_URL = r'(?:https?://)?vimeo.\com/album/(?P<id>\d+)'
_TITLE_RE = r'<header id="page_header">\n\s*<h1>(.*?)</h1>' _TITLE_RE = r'<header id="page_header">\n\s*<h1>(.*?)</h1>'
@ -336,12 +352,12 @@ class VimeoAlbumIE(VimeoChannelIE):
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
album_id = mobj.group('id') album_id = mobj.group('id')
return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id) return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id)
class VimeoGroupsIE(VimeoAlbumIE): class VimeoGroupsIE(VimeoAlbumIE):
IE_NAME = u'vimeo:group' IE_NAME = 'vimeo:group'
_VALID_URL = r'(?:https?://)?vimeo.\com/groups/(?P<name>[^/]+)' _VALID_URL = r'(?:https?://)?vimeo.\com/groups/(?P<name>[^/]+)'
def _extract_list_title(self, webpage): def _extract_list_title(self, webpage):
@ -351,3 +367,24 @@ class VimeoGroupsIE(VimeoAlbumIE):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
name = mobj.group('name') name = mobj.group('name')
return self._extract_videos(name, 'http://vimeo.com/groups/%s' % name) return self._extract_videos(name, 'http://vimeo.com/groups/%s' % name)
class VimeoReviewIE(InfoExtractor):
IE_NAME = 'vimeo:review'
IE_DESC = 'Review pages on vimeo'
_VALID_URL = r'(?:https?://)?vimeo.\com/[^/]+/review/(?P<id>[^/]+)'
_TEST = {
'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
'file': '75524534.mp4',
'md5': 'c507a72f780cacc12b2248bb4006d253',
'info_dict': {
'title': "DICK HARDWICK 'Comedian'",
'uploader': 'Richard Hardwick',
}
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
player_url = 'https://player.vimeo.com/player/' + video_id
return self.url_result(player_url, 'Vimeo', video_id)

1
youtube_dl/extractor/wistia.py
View File

@ -44,6 +44,7 @@ class WistiaIE(InfoExtractor):
'height': a['height'], 'height': a['height'],
'filesize': a['size'], 'filesize': a['size'],
'ext': a['ext'], 'ext': a['ext'],
'preference': 1 if atype == 'original' else None,
}) })
self._sort_formats(formats) self._sort_formats(formats)

32
youtube_dl/extractor/youporn.py
View File

@ -1,5 +1,4 @@
import json import json
import os
import re import re
import sys import sys
@ -16,6 +15,7 @@ from ..aes import (
aes_decrypt_text aes_decrypt_text
) )
class YouPornIE(InfoExtractor): class YouPornIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+))' _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+))'
_TEST = { _TEST = {
@ -71,38 +71,36 @@ class YouPornIE(InfoExtractor):
link = aes_decrypt_text(encrypted_link, video_title, 32).decode('utf-8') link = aes_decrypt_text(encrypted_link, video_title, 32).decode('utf-8')
links.append(link) links.append(link)
if not links:
raise ExtractorError(u'ERROR: no known formats available for video')
formats = [] formats = []
for link in links: for link in links:
# A link looks like this: # A link looks like this:
# http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0 # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0
# A path looks like this: # A path looks like this:
# /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4 # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4
video_url = unescapeHTML(link) video_url = unescapeHTML(link)
path = compat_urllib_parse_urlparse(video_url).path path = compat_urllib_parse_urlparse(video_url).path
extension = os.path.splitext(path)[1][1:] format_parts = path.split('/')[4].split('_')[:2]
format = path.split('/')[4].split('_')[:2]
# size = format[0] dn = compat_urllib_parse_urlparse(video_url).netloc.partition('.')[0]
# bitrate = format[1]
format = "-".join(format) resolution = format_parts[0]
# title = u'%s-%s-%s' % (video_title, size, bitrate) height = int(resolution[:-len('p')])
bitrate = int(format_parts[1][:-len('k')])
format = u'-'.join(format_parts) + u'-' + dn
formats.append({ formats.append({
'url': video_url, 'url': video_url,
'ext': extension,
'format': format, 'format': format,
'format_id': format, 'format_id': format,
'height': height,
'tbr': bitrate,
'resolution': resolution,
}) })
# Sort and remove doubles self._sort_formats(formats)
formats.sort(key=lambda format: list(map(lambda s: s.zfill(6), format['format'].split('-'))))
for i in range(len(formats)-1,0,-1): if not formats:
if formats[i]['format_id'] == formats[i-1]['format_id']: raise ExtractorError(u'ERROR: no known formats available for video')
del formats[i]
return { return {
'id': video_id, 'id': video_id,

14
youtube_dl/extractor/youtube.py
View File

@ -131,6 +131,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
( (
(?:https?://|//)? # http(s):// or protocol-independent URL (optional) (?:https?://|//)? # http(s):// or protocol-independent URL (optional)
(?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/| (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
(?:www\.)?deturl\.com/www\.youtube\.com/|
(?:www\.)?pwnyoutube\.com|
tube\.majestyc\.net/| tube\.majestyc\.net/|
youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
(?:.*?\#/)? # handle anchor (#/) redirect urls (?:.*?\#/)? # handle anchor (#/) redirect urls
@ -194,6 +196,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
'137': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40}, '137': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40},
'138': {'ext': 'mp4', 'height': 1081, 'resolution': '>1080p', 'format_note': 'DASH video', 'preference': -40}, '138': {'ext': 'mp4', 'height': 1081, 'resolution': '>1080p', 'format_note': 'DASH video', 'preference': -40},
'160': {'ext': 'mp4', 'height': 192, 'resolution': '192p', 'format_note': 'DASH video', 'preference': -40}, '160': {'ext': 'mp4', 'height': 192, 'resolution': '192p', 'format_note': 'DASH video', 'preference': -40},
'264': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40},
# Dash mp4 audio # Dash mp4 audio
'139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50}, '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
@ -212,6 +215,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
# Dash webm audio # Dash webm audio
'171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 48, 'preference': -50}, '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 48, 'preference': -50},
'172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 256, 'preference': -50}, '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 256, 'preference': -50},
# RTMP (unnamed)
'_rtmp': {'protocol': 'rtmp'},
} }
IE_NAME = u'youtube' IE_NAME = u'youtube'
@ -997,7 +1003,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
'lang': lang, 'lang': lang,
'v': video_id, 'v': video_id,
'fmt': self._downloader.params.get('subtitlesformat', 'srt'), 'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
'name': l[0].encode('utf-8'), 'name': unescapeHTML(l[0]).encode('utf-8'),
}) })
url = u'http://www.youtube.com/api/timedtext?' + params url = u'http://www.youtube.com/api/timedtext?' + params
sub_lang_list[lang] = url sub_lang_list[lang] = url
@ -1272,7 +1278,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'): if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
self.report_rtmp_download() self.report_rtmp_download()
video_url_list = [(None, video_info['conn'][0])] video_url_list = [('_rtmp', video_info['conn'][0])]
elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1: elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1:
encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts',[''])[0] encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts',[''])[0]
if 'rtmpe%3Dyes' in encoded_url_map: if 'rtmpe%3Dyes' in encoded_url_map:
@ -1758,6 +1764,6 @@ class YoutubeTruncatedURLIE(InfoExtractor):
u'Did you forget to quote the URL? Remember that & is a meta ' u'Did you forget to quote the URL? Remember that & is a meta '
u'character in most shells, so you want to put the URL in quotes, ' u'character in most shells, so you want to put the URL in quotes, '
u'like youtube-dl ' u'like youtube-dl '
u'\'http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc\'' u'"http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
u' (or simply youtube-dl BaW_jenozKc ).', u' or simply youtube-dl BaW_jenozKc .',
expected=True) expected=True)

18
youtube_dl/postprocessor/__init__.py Normal file
View File

@ -0,0 +1,18 @@
from .ffmpeg import (
FFmpegMergerPP,
FFmpegMetadataPP,
FFmpegVideoConvertor,
FFmpegExtractAudioPP,
FFmpegEmbedSubtitlePP,
)
from .xattrpp import XAttrMetadataPP
__all__ = [
'FFmpegMergerPP',
'FFmpegMetadataPP',
'FFmpegVideoConvertor',
'FFmpegExtractAudioPP',
'FFmpegEmbedSubtitlePP',
'XAttrMetadataPP',
]

49
youtube_dl/postprocessor/common.py Normal file
View File

@ -0,0 +1,49 @@
from ..utils import PostProcessingError
class PostProcessor(object):
"""Post Processor class.
PostProcessor objects can be added to downloaders with their
add_post_processor() method. When the downloader has finished a
successful download, it will take its internal chain of PostProcessors
and start calling the run() method on each one of them, first with
an initial argument and then with the returned value of the previous
PostProcessor.
The chain will be stopped if one of them ever returns None or the end
of the chain is reached.
PostProcessor objects follow a "mutual registration" process similar
to InfoExtractor objects.
"""
_downloader = None
def __init__(self, downloader=None):
self._downloader = downloader
def set_downloader(self, downloader):
"""Sets the downloader for this PP."""
self._downloader = downloader
def run(self, information):
"""Run the PostProcessor.
The "information" argument is a dictionary like the ones
composed by InfoExtractors. The only difference is that this
one has an extra field called "filepath" that points to the
downloaded file.
This method returns a tuple, the first element of which describes
whether the original file should be kept (i.e. not deleted - None for
no preference), and the second of which is the updated information.
In addition, this method may raise a PostProcessingError
exception if post processing fails.
"""
return None, information # by default, keep file and do nothing
class AudioConversionError(PostProcessingError):
pass

116
youtube_dl/PostProcessor.py → youtube_dl/postprocessor/ffmpeg.py
View File

@ -4,64 +4,23 @@ import sys
import time import time
from .utils import ( from .common import AudioConversionError, PostProcessor
from ..utils import (
check_executable,
compat_subprocess_get_DEVNULL, compat_subprocess_get_DEVNULL,
encodeFilename, encodeFilename,
PostProcessingError, PostProcessingError,
prepend_extension,
shell_quote, shell_quote,
subtitles_filename, subtitles_filename,
) )
class PostProcessor(object):
"""Post Processor class.
PostProcessor objects can be added to downloaders with their
add_post_processor() method. When the downloader has finished a
successful download, it will take its internal chain of PostProcessors
and start calling the run() method on each one of them, first with
an initial argument and then with the returned value of the previous
PostProcessor.
The chain will be stopped if one of them ever returns None or the end
of the chain is reached.
PostProcessor objects follow a "mutual registration" process similar
to InfoExtractor objects.
"""
_downloader = None
def __init__(self, downloader=None):
self._downloader = downloader
def set_downloader(self, downloader):
"""Sets the downloader for this PP."""
self._downloader = downloader
def run(self, information):
"""Run the PostProcessor.
The "information" argument is a dictionary like the ones
composed by InfoExtractors. The only difference is that this
one has an extra field called "filepath" that points to the
downloaded file.
This method returns a tuple, the first element of which describes
whether the original file should be kept (i.e. not deleted - None for
no preference), and the second of which is the updated information.
In addition, this method may raise a PostProcessingError
exception if post processing fails.
"""
return None, information # by default, keep file and do nothing
class FFmpegPostProcessorError(PostProcessingError): class FFmpegPostProcessorError(PostProcessingError):
pass pass
class AudioConversionError(PostProcessingError):
pass
class FFmpegPostProcessor(PostProcessor): class FFmpegPostProcessor(PostProcessor):
def __init__(self,downloader=None): def __init__(self,downloader=None):
PostProcessor.__init__(self, downloader) PostProcessor.__init__(self, downloader)
@ -69,25 +28,28 @@ class FFmpegPostProcessor(PostProcessor):
@staticmethod @staticmethod
def detect_executables(): def detect_executables():
def executable(exe):
try:
subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
except OSError:
return False
return exe
programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe'] programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
return dict((program, executable(program)) for program in programs) return dict((program, check_executable(program, ['-version'])) for program in programs)
def _get_executable(self):
if self._downloader.params.get('prefer_ffmpeg', False):
return self._exes['ffmpeg'] or self._exes['avconv']
else:
return self._exes['avconv'] or self._exes['ffmpeg']
def _uses_avconv(self):
return self._get_executable() == self._exes['avconv']
def run_ffmpeg_multiple_files(self, input_paths, out_path, opts): def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
if not self._exes['ffmpeg'] and not self._exes['avconv']: if not self._get_executable():
raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.') raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.')
files_cmd = [] files_cmd = []
for path in input_paths: for path in input_paths:
files_cmd.extend(['-i', encodeFilename(path)]) files_cmd.extend(['-i', encodeFilename(path, True)])
cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y'] + files_cmd cmd = ([self._get_executable(), '-y'] + files_cmd
+ opts + + opts +
[encodeFilename(self._ffmpeg_filename_argument(out_path))]) [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
if self._downloader.params.get('verbose', False): if self._downloader.params.get('verbose', False):
self._downloader.to_screen(u'[debug] ffmpeg command line: %s' % shell_quote(cmd)) self._downloader.to_screen(u'[debug] ffmpeg command line: %s' % shell_quote(cmd))
@ -107,6 +69,7 @@ class FFmpegPostProcessor(PostProcessor):
return u'./' + fn return u'./' + fn
return fn return fn
class FFmpegExtractAudioPP(FFmpegPostProcessor): class FFmpegExtractAudioPP(FFmpegPostProcessor):
def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False): def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False):
FFmpegPostProcessor.__init__(self, downloader) FFmpegPostProcessor.__init__(self, downloader)
@ -120,7 +83,10 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
if not self._exes['ffprobe'] and not self._exes['avprobe']: if not self._exes['ffprobe'] and not self._exes['avprobe']:
raise PostProcessingError(u'ffprobe or avprobe not found. Please install one.') raise PostProcessingError(u'ffprobe or avprobe not found. Please install one.')
try: try:
cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', encodeFilename(self._ffmpeg_filename_argument(path))] cmd = [
self._exes['avprobe'] or self._exes['ffprobe'],
'-show_streams',
encodeFilename(self._ffmpeg_filename_argument(path), True)]
handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE) handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
output = handle.communicate()[0] output = handle.communicate()[0]
if handle.wait() != 0: if handle.wait() != 0:
@ -136,8 +102,6 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
return None return None
def run_ffmpeg(self, path, out_path, codec, more_opts): def run_ffmpeg(self, path, out_path, codec, more_opts):
if not self._exes['ffmpeg'] and not self._exes['avconv']:
raise AudioConversionError('ffmpeg or avconv not found. Please install one.')
if codec is None: if codec is None:
acodec_opts = [] acodec_opts = []
else: else:
@ -155,13 +119,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
if filecodec is None: if filecodec is None:
raise PostProcessingError(u'WARNING: unable to obtain file audio codec with ffprobe') raise PostProcessingError(u'WARNING: unable to obtain file audio codec with ffprobe')
uses_avconv = self._uses_avconv()
more_opts = [] more_opts = []
if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'): if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']: if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']:
# Lossless, but in another container # Lossless, but in another container
acodec = 'copy' acodec = 'copy'
extension = 'm4a' extension = 'm4a'
more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc'] more_opts = ['-bsf:a' if uses_avconv else '-absf', 'aac_adtstoasc']
elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']: elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
# Lossless if possible # Lossless if possible
acodec = 'copy' acodec = 'copy'
@ -177,9 +142,9 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
more_opts = [] more_opts = []
if self._preferredquality is not None: if self._preferredquality is not None:
if int(self._preferredquality) < 10: if int(self._preferredquality) < 10:
more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality] more_opts += ['-q:a' if uses_avconv else '-aq', self._preferredquality]
else: else:
more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k'] more_opts += ['-b:a' if uses_avconv else '-ab', self._preferredquality + 'k']
else: else:
# We convert the audio (lossy) # We convert the audio (lossy)
acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec] acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
@ -188,13 +153,13 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
if self._preferredquality is not None: if self._preferredquality is not None:
# The opus codec doesn't support the -aq option # The opus codec doesn't support the -aq option
if int(self._preferredquality) < 10 and extension != 'opus': if int(self._preferredquality) < 10 and extension != 'opus':
more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality] more_opts += ['-q:a' if uses_avconv else '-aq', self._preferredquality]
else: else:
more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k'] more_opts += ['-b:a' if uses_avconv else '-ab', self._preferredquality + 'k']
if self._preferredcodec == 'aac': if self._preferredcodec == 'aac':
more_opts += ['-f', 'adts'] more_opts += ['-f', 'adts']
if self._preferredcodec == 'm4a': if self._preferredcodec == 'm4a':
more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc'] more_opts += ['-bsf:a' if uses_avconv else '-absf', 'aac_adtstoasc']
if self._preferredcodec == 'vorbis': if self._preferredcodec == 'vorbis':
extension = 'ogg' extension = 'ogg'
if self._preferredcodec == 'wav': if self._preferredcodec == 'wav':
@ -212,14 +177,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)): if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
self._downloader.to_screen(u'[youtube] Post-process file %s exists, skipping' % new_path) self._downloader.to_screen(u'[youtube] Post-process file %s exists, skipping' % new_path)
else: else:
self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path) self._downloader.to_screen(u'[' + self._get_executable() + '] Destination: ' + new_path)
self.run_ffmpeg(path, new_path, acodec, more_opts) self.run_ffmpeg(path, new_path, acodec, more_opts)
except: except:
etype,e,tb = sys.exc_info() etype,e,tb = sys.exc_info()
if isinstance(e, AudioConversionError): if isinstance(e, AudioConversionError):
msg = u'audio conversion failed: ' + e.msg msg = u'audio conversion failed: ' + e.msg
else: else:
msg = u'error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') msg = u'error running ' + self._get_executable()
raise PostProcessingError(msg) raise PostProcessingError(msg)
# Try to update the date time for extracted audio file. # Try to update the date time for extracted audio file.
@ -232,6 +197,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
information['filepath'] = new_path information['filepath'] = new_path
return self._nopostoverwrites,information return self._nopostoverwrites,information
class FFmpegVideoConvertor(FFmpegPostProcessor): class FFmpegVideoConvertor(FFmpegPostProcessor):
def __init__(self, downloader=None,preferedformat=None): def __init__(self, downloader=None,preferedformat=None):
super(FFmpegVideoConvertor, self).__init__(downloader) super(FFmpegVideoConvertor, self).__init__(downloader)
@ -496,16 +462,24 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
return True, info return True, info
filename = info['filepath'] filename = info['filepath']
ext = os.path.splitext(filename)[1][1:] temp_filename = prepend_extension(filename, 'temp')
temp_filename = filename + u'.temp'
options = ['-c', 'copy'] options = ['-c', 'copy']
for (name, value) in metadata.items(): for (name, value) in metadata.items():
options.extend(['-metadata', '%s=%s' % (name, value)]) options.extend(['-metadata', '%s=%s' % (name, value)])
options.extend(['-f', ext])
self._downloader.to_screen(u'[ffmpeg] Adding metadata to \'%s\'' % filename) self._downloader.to_screen(u'[ffmpeg] Adding metadata to \'%s\'' % filename)
self.run_ffmpeg(filename, temp_filename, options) self.run_ffmpeg(filename, temp_filename, options)
os.remove(encodeFilename(filename)) os.remove(encodeFilename(filename))
os.rename(encodeFilename(temp_filename), encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename))
return True, info return True, info
class FFmpegMergerPP(FFmpegPostProcessor):
def run(self, info):
filename = info['filepath']
args = ['-c', 'copy']
self._downloader.to_screen(u'[ffmpeg] Merging formats into "%s"' % filename)
self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
return True, info

108
youtube_dl/postprocessor/xattrpp.py Normal file
View File

@ -0,0 +1,108 @@
import os
import subprocess
import sys
from .common import PostProcessor
from ..utils import (
check_executable,
hyphenate_date,
)
class XAttrMetadataPP(PostProcessor):
#
# More info about extended attributes for media:
# http://freedesktop.org/wiki/CommonExtendedAttributes/
# http://www.freedesktop.org/wiki/PhreedomDraft/
# http://dublincore.org/documents/usageguide/elements.shtml
#
# TODO:
# * capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated)
# * figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution'
#
def run(self, info):
""" Set extended attributes on downloaded file (if xattr support is found). """
# This mess below finds the best xattr tool for the job and creates a
# "write_xattr" function.
try:
# try the pyxattr module...
import xattr
def write_xattr(path, key, value):
return xattr.setxattr(path, key, value)
except ImportError:
if os.name == 'nt':
# Write xattrs to NTFS Alternate Data Streams:
# http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
def write_xattr(path, key, value):
assert ':' not in key
assert os.path.exists(path)
ads_fn = path + ":" + key
with open(ads_fn, "wb") as f:
f.write(value)
else:
user_has_setfattr = check_executable("setfattr", ['--version'])
user_has_xattr = check_executable("xattr", ['-h'])
if user_has_setfattr or user_has_xattr:
def write_xattr(path, key, value):
if user_has_setfattr:
cmd = ['setfattr', '-n', key, '-v', value, path]
elif user_has_xattr:
cmd = ['xattr', '-w', key, value, path]
subprocess.check_output(cmd)
else:
# On Unix, and can't find pyxattr, setfattr, or xattr.
if sys.platform.startswith('linux'):
self._downloader.report_error(
"Couldn't find a tool to set the xattrs. "
"Install either the python 'pyxattr' or 'xattr' "
"modules, or the GNU 'attr' package "
"(which contains the 'setfattr' tool).")
else:
self._downloader.report_error(
"Couldn't find a tool to set the xattrs. "
"Install either the python 'xattr' module, "
"or the 'xattr' binary.")
# Write the metadata to the file's xattrs
self._downloader.to_screen('[metadata] Writing metadata to file\'s xattrs')
filename = info['filepath']
try:
xattr_mapping = {
'user.xdg.referrer.url': 'webpage_url',
# 'user.xdg.comment': 'description',
'user.dublincore.title': 'title',
'user.dublincore.date': 'upload_date',
'user.dublincore.description': 'description',
'user.dublincore.contributor': 'uploader',
'user.dublincore.format': 'format',
}
for xattrname, infoname in xattr_mapping.items():
value = info.get(infoname)
if value:
if infoname == "upload_date":
value = hyphenate_date(value)
byte_value = value.encode('utf-8')
write_xattr(filename, xattrname, byte_value)
return True, info
except (subprocess.CalledProcessError, OSError):
self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
return False, info

71
youtube_dl/utils.py
View File

@ -500,12 +500,13 @@ def unescapeHTML(s):
result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s) result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
return result return result
def encodeFilename(s):
def encodeFilename(s, for_subprocess=False):
""" """
@param s The name of the file @param s The name of the file
""" """
assert type(s) == type(u'') assert type(s) == compat_str
# Python 3 has a Unicode API # Python 3 has a Unicode API
if sys.version_info >= (3, 0): if sys.version_info >= (3, 0):
@ -515,12 +516,18 @@ def encodeFilename(s):
# Pass u'' directly to use Unicode APIs on Windows 2000 and up # Pass u'' directly to use Unicode APIs on Windows 2000 and up
# (Detecting Windows NT 4 is tricky because 'major >= 4' would # (Detecting Windows NT 4 is tricky because 'major >= 4' would
# match Windows 9x series as well. Besides, NT 4 is obsolete.) # match Windows 9x series as well. Besides, NT 4 is obsolete.)
return s if not for_subprocess:
return s
else:
# For subprocess calls, encode with locale encoding
# Refer to http://stackoverflow.com/a/9951851/35070
encoding = preferredencoding()
else: else:
encoding = sys.getfilesystemencoding() encoding = sys.getfilesystemencoding()
if encoding is None: if encoding is None:
encoding = 'utf-8' encoding = 'utf-8'
return s.encode(encoding, 'ignore') return s.encode(encoding, 'ignore')
def decodeOption(optval): def decodeOption(optval):
if optval is None: if optval is None:
@ -539,7 +546,8 @@ def formatSeconds(secs):
else: else:
return '%d' % secs return '%d' % secs
def make_HTTPS_handler(opts_no_check_certificate):
def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
if sys.version_info < (3, 2): if sys.version_info < (3, 2):
import httplib import httplib
@ -560,7 +568,7 @@ def make_HTTPS_handler(opts_no_check_certificate):
class HTTPSHandlerV3(compat_urllib_request.HTTPSHandler): class HTTPSHandlerV3(compat_urllib_request.HTTPSHandler):
def https_open(self, req): def https_open(self, req):
return self.do_open(HTTPSConnectionV3, req) return self.do_open(HTTPSConnectionV3, req)
return HTTPSHandlerV3() return HTTPSHandlerV3(**kwargs)
else: else:
context = ssl.SSLContext(ssl.PROTOCOL_SSLv3) context = ssl.SSLContext(ssl.PROTOCOL_SSLv3)
context.verify_mode = (ssl.CERT_NONE context.verify_mode = (ssl.CERT_NONE
@ -571,7 +579,7 @@ def make_HTTPS_handler(opts_no_check_certificate):
context.load_default_certs() context.load_default_certs()
except AttributeError: except AttributeError:
pass # Python < 3.4 pass # Python < 3.4
return compat_urllib_request.HTTPSHandler(context=context) return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
class ExtractorError(Exception): class ExtractorError(Exception):
"""Error during info extraction.""" """Error during info extraction."""
@ -756,6 +764,7 @@ def unified_strdate(date_str):
'%Y-%m-%d', '%Y-%m-%d',
'%d/%m/%Y', '%d/%m/%Y',
'%Y/%m/%d %H:%M:%S', '%Y/%m/%d %H:%M:%S',
'%Y-%m-%d %H:%M:%S',
'%d.%m.%Y %H:%M', '%d.%m.%Y %H:%M',
'%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%SZ',
'%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%S.%fZ',
@ -809,6 +818,15 @@ def date_from_str(date_str):
return today + delta return today + delta
return datetime.datetime.strptime(date_str, "%Y%m%d").date() return datetime.datetime.strptime(date_str, "%Y%m%d").date()
def hyphenate_date(date_str):
"""
Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
if match is not None:
return '-'.join(match.groups())
else:
return date_str
class DateRange(object): class DateRange(object):
"""Represents a time interval between two dates""" """Represents a time interval between two dates"""
def __init__(self, start=None, end=None): def __init__(self, start=None, end=None):
@ -849,12 +867,22 @@ def platform_name():
def write_string(s, out=None): def write_string(s, out=None):
if out is None: if out is None:
out = sys.stderr out = sys.stderr
assert type(s) == type(u'') assert type(s) == compat_str
if ('b' in getattr(out, 'mode', '') or if ('b' in getattr(out, 'mode', '') or
sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
s = s.encode(preferredencoding(), 'ignore') s = s.encode(preferredencoding(), 'ignore')
out.write(s) try:
out.write(s)
except UnicodeEncodeError:
# In Windows shells, this can fail even when the codec is just charmap!?
# See https://wiki.python.org/moin/PrintFails#Issue
if sys.platform == 'win32' and hasattr(out, 'encoding'):
s = s.encode(out.encoding, 'ignore').decode(out.encoding)
out.write(s)
else:
raise
out.flush() out.flush()
@ -1008,9 +1036,9 @@ def smuggle_url(url, data):
return url + u'#' + sdata return url + u'#' + sdata
def unsmuggle_url(smug_url): def unsmuggle_url(smug_url, default=None):
if not '#__youtubedl_smuggle' in smug_url: if not '#__youtubedl_smuggle' in smug_url:
return smug_url, None return smug_url, default
url, _, sdata = smug_url.rpartition(u'#') url, _, sdata = smug_url.rpartition(u'#')
jsond = compat_parse_qs(sdata)[u'__youtubedl_smuggle'][0] jsond = compat_parse_qs(sdata)[u'__youtubedl_smuggle'][0]
data = json.loads(jsond) data = json.loads(jsond)
@ -1070,7 +1098,7 @@ def fix_xml_all_ampersand(xml_str):
def setproctitle(title): def setproctitle(title):
assert isinstance(title, type(u'')) assert isinstance(title, compat_str)
try: try:
libc = ctypes.cdll.LoadLibrary("libc.so.6") libc = ctypes.cdll.LoadLibrary("libc.so.6")
except OSError: except OSError:
@ -1118,3 +1146,18 @@ def parse_duration(s):
if m.group('hours'): if m.group('hours'):
res += int(m.group('hours')) * 60 * 60 res += int(m.group('hours')) * 60 * 60
return res return res
def prepend_extension(filename, ext):
name, real_ext = os.path.splitext(filename)
return u'{0}.{1}{2}'.format(name, ext, real_ext)
def check_executable(exe, args=[]):
""" Checks if the given binary is installed somewhere in PATH, and returns its name.
args can be a list of arguments for a short output (like -version) """
try:
subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
except OSError:
return False
return exe

2
youtube_dl/version.py
View File

@ -1,2 +1,2 @@
__version__ = '2013.12.23.4' __version__ = '2014.01.17.2'