diff --git a/.gitignore b/.gitignore index 86312d4e4..0422adf44 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ updates_key.pem test/testdata .tox youtube-dl.zsh +.idea +.idea/* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index c6cc7a994..fb34299fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,14 @@ python: - "2.7" - "3.3" - "3.4" +before_install: + - sudo apt-get update -qq + - sudo apt-get install -yqq rtmpdump script: nosetests test --verbose notifications: email: - filippo.valsorda@gmail.com - phihag@phihag.de - - jaime.marquinez.ferrandiz+travis@gmail.com - yasoob.khld@gmail.com # irc: # channels: diff --git a/AUTHORS b/AUTHORS index d4b0b75fa..47f12a9ee 100644 --- a/AUTHORS +++ b/AUTHORS @@ -88,3 +88,26 @@ Dao Hoang Son Oskar Jauch Matthew Rayfield t0mm0 +Tithen-Firion +Zack Fernandes +cryptonaut +Adrian Kretz +Mathias Rav +Petr Kutalek +Will Glynn +Max Reimann +Cédric Luthi +Thijs Vermeir +Joel Leclerc +Christopher Krooss +Ondřej Caletka +Dinesh S +Johan K. Jensen +Yen Chi Hsuan +Enam Mijbah Noor +David Luhmer +Shaya Goldberg +Paul Hartmann +Frans de Jonge +Robin de Rooij +Ryan Schmidt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..351229f21 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,138 @@ +**Please include the full output of youtube-dl when run with `-v`**. + +The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever. + +Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist): + +### Is the description of the issue itself sufficient? + +We often get issue reports that we cannot really decipher. While in most cases we eventually get the required information after asking back multiple times, this poses an unnecessary drain on our resources. Many contributors, including myself, are also not native speakers, so we may misread some parts. + +So please elaborate on what feature you are requesting, or what bug you want to be fixed. Make sure that it's obvious + +- What the problem is +- How it could be fixed +- How your proposed solution would look like + +If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over. + +For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information. + +Site support requests **must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL. + +### Are you using the latest version? + +Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well. + +### Is the issue already documented? + +Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or at https://github.com/rg3/youtube-dl/search?type=Issues . If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity. + +### Why are existing options not enough? + +Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/rg3/youtube-dl/blob/master/README.md#synopsis). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem. + +### Is there enough context in your bug report? + +People want to solve problems, and often think they do us a favor by breaking down their larger problems (e.g. wanting to skip already downloaded files) to a specific request (e.g. requesting us to look whether the file exists before downloading the info page). However, what often happens is that they break down the problem into two steps: One simple, and one impossible (or extremely complicated one). + +We are then presented with a very complicated request when the original problem could be solved far easier, e.g. by recording the downloaded video IDs in a separate file. To avoid this, you must include the greater context where it is non-obvious. In particular, every feature request that does not consist of adding support for a new site should contain a use case scenario that explains in what situation the missing feature would be useful. + +### Does the issue involve one problem, and one problem only? + +Some of our users seem to think there is a limit of issues they can or should open. There is no limit of issues they can or should open. While it may seem appealing to be able to dump all your issues into one ticket, that means that someone who solves one of your issues cannot mark the issue as closed. Typically, reporting a bunch of issues leads to the ticket lingering since nobody wants to attack that behemoth, until someone mercifully splits the issue into multiple ones. + +In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of youtube-dl that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service. + +### Is anyone going to need the feature? + +Only post features that you (or an incapacitated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them. + +### Is your question about youtube-dl? + +It may sound strange, but some bug reports we receive are completely unrelated to youtube-dl and relate to a different or even the reporter's own application. Please make sure that you are actually using youtube-dl. If you are using a UI for youtube-dl, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for youtube-dl fails in some way you believe is related to youtube-dl, by all means, go ahead and report the bug. + +# DEVELOPER INSTRUCTIONS + +Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution. + +To run youtube-dl as a developer, you don't need to build anything either. Simply execute + + python -m youtube_dl + +To run the test, simply invoke your favorite test runner, or execute a test file directly; any of the following work: + + python -m unittest discover + python test/test_download.py + nosetests + +If you want to create a build of youtube-dl yourself, you'll need + +* python +* make +* pandoc +* zip +* nosetests + +### Adding support for a new site + +If you want to add support for a new site, you can follow this quick list (assuming your service is called `yourextractor`): + +1. [Fork this repository](https://github.com/rg3/youtube-dl/fork) +2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git` +3. Start a new git branch with `cd youtube-dl; git checkout -b yourextractor` +4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`: + ```python + # coding: utf-8 + from __future__ import unicode_literals + + from .common import InfoExtractor + + + class YourExtractorIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P[0-9]+)' + _TEST = { + 'url': 'http://yourextractor.com/watch/42', + 'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)', + 'info_dict': { + 'id': '42', + 'ext': 'mp4', + 'title': 'Video title goes here', + 'thumbnail': 're:^https?://.*\.jpg$', + # TODO more properties, either as: + # * A value + # * MD5 checksum; start the string with md5: + # * A regular expression; start the string with re: + # * Any Python type (for example int or float) + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + # TODO more code goes here, for example ... + title = self._html_search_regex(r'

(.*?)

', webpage, 'title') + + return { + 'id': video_id, + 'title': title, + 'description': self._og_search_description(webpage), + # TODO more properties (see youtube_dl/extractor/common.py) + } + ``` +5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py). +6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. +7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want. +8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8). +9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this: + + $ git add youtube_dl/extractor/__init__.py + $ git add youtube_dl/extractor/yourextractor.py + $ git commit -m '[yourextractor] Add new extractor' + $ git push origin yourextractor + +10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it. + +In any case, thank you very much for your contributions! + diff --git a/Makefile b/Makefile index 3e1debc7e..573c82685 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,7 @@ -all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish +all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites clean: - rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part - -cleanall: clean - rm -f youtube-dl youtube-dl.exe + rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe PREFIX ?= /usr/local BINDIR ?= $(PREFIX)/bin @@ -35,13 +32,22 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtu install -d $(DESTDIR)$(SYSCONFDIR)/fish/completions install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish +codetest: + flake8 . + test: #nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test nosetests --verbose test + $(MAKE) codetest + +ot: offlinetest + +offlinetest: codetest + nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles --exclude test_write_annotations --exclude test_youtube_lists tar: youtube-dl.tar.gz -.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion +.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest supportedsites pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish @@ -54,7 +60,13 @@ youtube-dl: youtube_dl/*.py youtube_dl/*/*.py chmod a+x youtube-dl README.md: youtube_dl/*.py youtube_dl/*/*.py - COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py + COLUMNS=80 python youtube_dl/__main__.py --help | python devscripts/make_readme.py + +CONTRIBUTING.md: README.md + python devscripts/make_contributing.py README.md CONTRIBUTING.md + +supportedsites: + python devscripts/make_supportedsites.py docs/supportedsites.md README.txt: README.md pandoc -f markdown -t plain README.md -o README.txt diff --git a/README.md b/README.md index d6e7ff902..8ea31d605 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,15 @@ youtube-dl - download videos from youtube.com or other video platforms -# SYNOPSIS -**youtube-dl** [OPTIONS] URL [URL...] +- [INSTALLATION](#installation) +- [DESCRIPTION](#description) +- [OPTIONS](#options) +- [CONFIGURATION](#configuration) +- [OUTPUT TEMPLATE](#output-template) +- [VIDEO SELECTION](#video-selection) +- [FAQ](#faq) +- [DEVELOPER INSTRUCTIONS](#developer-instructions) +- [BUGS](#bugs) +- [COPYRIGHT](#copyright) # INSTALLATION @@ -34,6 +42,8 @@ YouTube.com and a few more sites. It requires the Python interpreter, version your Unix box, on Windows or on Mac OS X. It is released to the public domain, which means you can modify it, redistribute it or use it however you like. + youtube-dl [OPTIONS] URL [URL...] + # OPTIONS -h, --help print this help text and exit --version print program version and exit @@ -50,10 +60,6 @@ which means you can modify it, redistribute it or use it however you like. they would handle --extractor-descriptions Output descriptions of all supported extractors - --proxy URL Use the specified HTTP/HTTPS proxy. Pass in - an empty string (--proxy "") for direct - connection - --socket-timeout None Time to wait before giving up, in seconds --default-search PREFIX Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large @@ -65,16 +71,37 @@ which means you can modify it, redistribute it or use it however you like. this is not possible instead of searching. --ignore-config Do not read configuration files. When given in the global configuration file /etc - /youtube-dl.conf: do not read the user - configuration in ~/.config/youtube-dl.conf - (%APPDATA%/youtube-dl/config.txt on - Windows) + /youtube-dl.conf: Do not read the user + configuration in ~/.config/youtube- + dl/config (%APPDATA%/youtube-dl/config.txt + on Windows) --flat-playlist Do not extract the videos of a playlist, only list them. + --no-color Do not emit color codes in output. + +## Network Options: + --proxy URL Use the specified HTTP/HTTPS proxy. Pass in + an empty string (--proxy "") for direct + connection + --socket-timeout SECONDS Time to wait before giving up, in seconds + --source-address IP Client-side IP address to bind to + (experimental) + -4, --force-ipv4 Make all connections via IPv4 + (experimental) + -6, --force-ipv6 Make all connections via IPv6 + (experimental) ## Video Selection: --playlist-start NUMBER playlist video to start at (default is 1) --playlist-end NUMBER playlist video to end at (default is last) + --playlist-items ITEM_SPEC playlist video items to download. Specify + indices of the videos in the playlist + seperated by commas like: "--playlist-items + 1,2,5,8" if you want to download videos + indexed 1, 2, 5, 8 in the playlist. You can + specify range: "--playlist-items + 1-3,7,10-13", it will download the videos + at index 1, 2, 3, 7, 10, 11, 12 and 13. --match-title REGEX download only matching titles (regex or caseless sub-string) --reject-title REGEX skip download for matching titles (regex or @@ -93,6 +120,23 @@ which means you can modify it, redistribute it or use it however you like. COUNT views --max-views COUNT Do not download any videos with more than COUNT views + --match-filter FILTER (Experimental) Generic video filter. + Specify any key (see help for -o for a list + of available keys) to match if the key is + present, !key to check if the key is not + present,key > NUMBER (like "comment_count > + 12", also works with >=, <, <=, !=, =) to + compare against a number, and & to require + multiple matches. Values which are not + known are excluded unless you put a + question mark (?) after the operator.For + example, to only match videos that have + been liked more than 100 times and disliked + less than 50 times (or the dislike + functionality is not available at the given + service), but who also have a description, + use --match-filter "like_count > 100 & + dislike_count 10M]"). This works for + filesize, height, width, tbr, abr, vbr, + asr, and fps and the comparisons <, <=, >, + >=, =, != and for ext, acodec, vcodec, + container, and protocol and the comparisons + =, != . Formats for which the value is not + known are excluded unless you put a + question mark (?) after the operator. You + can combine format filters, so -f "[height + <=? 720][tbr>500]" selects up to 720p + videos (or videos where the height is not + known) with a bitrate of at least 500 + KBit/s. By default, youtube-dl will pick + the best quality. Use commas to download + multiple audio formats, such as -f 136/137/mp4/bestvideo,140/m4a/bestaudio. You can merge the video and audio of two formats into a single file using -f \youtube-dl.conf`. +You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\\youtube-dl.conf`. # OUTPUT TEMPLATE @@ -407,17 +503,27 @@ Apparently YouTube requires you to pass a CAPTCHA test if you download too much. Once the video is fully downloaded, use any video player, such as [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/). -### The links provided by youtube-dl -g are not working anymore +### I extracted a video URL with -g, but it does not play on another machine / in my webbrowser. -The URLs youtube-dl outputs require the downloader to have the correct cookies. Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl. +It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies. Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl. + +It may be beneficial to use IPv6; in some cases, the restrictions are only applied to IPv4. Some services (sometimes only for a subset of videos) do not restrict the video URL by IP address, cookie, or user-agent, but these are the exception rather than the rule. + +Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using -g, your own downloader must support these as well. + +If you want to play the video on a machine that is not running youtube-dl, you can relay the video content from the machine that runs youtube-dl. You can use `-o -` to let youtube-dl stream a video to stdout, or simply allow the player to download the files written by youtube-dl in turn. ### ERROR: no fmt_url_map or conn information found in video info -youtube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`. +YouTube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl. ### ERROR: unable to download video ### -youtube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`. +YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl. + +### ExtractorError: Could not find JS function u'OF' + +In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl. ### SyntaxError: Non-ASCII character ### @@ -436,6 +542,41 @@ Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unz To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistributable Package](http://www.microsoft.com/en-us/download/details.aspx?id=29). +### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files? + +If you put youtube-dl and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome. + +To make a different directory work - either for ffmpeg, or for youtube-dl, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory. + +From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in. + +### How do I put downloads into a specific folder? + +Use the `-o` to specify an [output template](#output-template), for example `-o "/home/user/videos/%(title)s-%(id)s.%(ext)s"`. If you want this for all of your downloads, put the option into your [configuration file](#configuration). + +### How do I download a video starting with a `-` ? + +Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the options with `--`: + + youtube-dl -- -wNyEUrxzFU + youtube-dl "http://www.youtube.com/watch?v=-wNyEUrxzFU" + +### Can you add support for this anime video site, or site which shows current movies for free? + +As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl. + +A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into youtube-dl. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization. + +Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content. + +### How can I detect whether a given URL is supported by youtube-dl? + +For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug. + +It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor. + +If you want to find out whether a given URL is supported, simply call youtube-dl with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program. + # DEVELOPER INSTRUCTIONS Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution. @@ -508,7 +649,7 @@ If you want to add support for a new site, you can follow this quick list (assum 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py). 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. 7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want. -8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501). +8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8). 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this: $ git add youtube_dl/extractor/__init__.py @@ -526,23 +667,61 @@ youtube-dl makes the best effort to be a good command-line program, and thus sho From a Python program, you can embed youtube-dl in a more powerful fashion, like this: - import youtube_dl +```python +import youtube_dl - ydl_opts = {} - with youtube_dl.YoutubeDL(ydl_opts) as ydl: - ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc']) +ydl_opts = {} +with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc']) +``` Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object. +Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file: + +```python +import youtube_dl + + +class MyLogger(object): + def debug(self, msg): + pass + + def warning(self, msg): + pass + + def error(self, msg): + print(msg) + + +def my_hook(d): + if d['status'] == 'finished': + print('Done downloading, now converting ...') + + +ydl_opts = { + 'format': 'bestaudio/best', + 'postprocessors': [{ + 'key': 'FFmpegExtractAudio', + 'preferredcodec': 'mp3', + 'preferredquality': '192', + }], + 'logger': MyLogger(), + 'progress_hooks': [my_hook], +} +with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc']) +``` + # BUGS -Bugs and suggestions should be reported at: . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. +Bugs and suggestions should be reported at: . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode. -Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever. +**Please include the full output of youtube-dl when run with `-v`**. -For discussions, join us in the irc channel #youtube-dl on freenode. +The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever. -When you submit a request, please re-read it once to avoid a couple of mistakes (you can and should use this as a checklist): +Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist): ### Is the description of the issue itself sufficient? @@ -586,7 +765,7 @@ In particular, every site support request issue should only pertain to services ### Is anyone going to need the feature? -Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them. +Only post features that you (or an incapacitated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them. ### Is your question about youtube-dl? diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py index f0f0481c7..d3ef5f0b5 100755 --- a/devscripts/gh-pages/update-sites.py +++ b/devscripts/gh-pages/update-sites.py @@ -16,7 +16,7 @@ def main(): template = tmplf.read() ie_htmls = [] - for ie in sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower()): + for ie in youtube_dl.list_extractors(age_limit=None): ie_html = '{}'.format(ie.IE_NAME) ie_desc = getattr(ie, 'IE_DESC', None) if ie_desc is False: diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py new file mode 100755 index 000000000..5e454a429 --- /dev/null +++ b/devscripts/make_contributing.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +import io +import optparse +import re + + +def main(): + parser = optparse.OptionParser(usage='%prog INFILE OUTFILE') + options, args = parser.parse_args() + if len(args) != 2: + parser.error('Expected an input and an output filename') + + infile, outfile = args + + with io.open(infile, encoding='utf-8') as inf: + readme = inf.read() + + bug_text = re.search( + r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1) + dev_text = re.search( + r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*EMBEDDING YOUTUBE-DL', + readme).group(1) + + out = bug_text + dev_text + + with io.open(outfile, 'w', encoding='utf-8') as outf: + outf.write(out) + +if __name__ == '__main__': + main() diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py new file mode 100644 index 000000000..3df4385a6 --- /dev/null +++ b/devscripts/make_supportedsites.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +import io +import optparse +import os +import sys + + +# Import youtube_dl +ROOT_DIR = os.path.join(os.path.dirname(__file__), '..') +sys.path.append(ROOT_DIR) +import youtube_dl + + +def main(): + parser = optparse.OptionParser(usage='%prog OUTFILE.md') + options, args = parser.parse_args() + if len(args) != 1: + parser.error('Expected an output filename') + + outfile, = args + + def gen_ies_md(ies): + for ie in ies: + ie_md = '**{0}**'.format(ie.IE_NAME) + ie_desc = getattr(ie, 'IE_DESC', None) + if ie_desc is False: + continue + if ie_desc is not None: + ie_md += ': {0}'.format(ie.IE_DESC) + if not ie.working(): + ie_md += ' (Currently broken)' + yield ie_md + + ies = sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower()) + out = '# Supported sites\n' + ''.join( + ' - ' + md + '\n' + for md in gen_ies_md(ies)) + + with io.open(outfile, 'w', encoding='utf-8') as outf: + outf.write(out) + +if __name__ == '__main__': + main() diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py index f66bebfea..7ece37754 100644 --- a/devscripts/prepare_manpage.py +++ b/devscripts/prepare_manpage.py @@ -11,8 +11,19 @@ README_FILE = os.path.join(ROOT_DIR, 'README.md') with io.open(README_FILE, encoding='utf-8') as f: readme = f.read() -PREFIX = '%YOUTUBE-DL(1)\n\n# NAME\n' -readme = re.sub(r'(?s)# INSTALLATION.*?(?=# DESCRIPTION)', '', readme) +PREFIX = '''%YOUTUBE-DL(1) + +# NAME + +youtube\-dl \- download videos from youtube.com or other video platforms + +# SYNOPSIS + +**youtube-dl** \[OPTIONS\] URL [URL...] + +''' +readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme) +readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme) readme = PREFIX + readme if sys.version_info < (3, 0): diff --git a/devscripts/release.sh b/devscripts/release.sh index 691517ceb..61806961c 100755 --- a/devscripts/release.sh +++ b/devscripts/release.sh @@ -35,7 +35,7 @@ if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $us if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi /bin/echo -e "\n### First of all, testing..." -make cleanall +make clean if $skip_tests ; then echo 'SKIPPING TESTS' else @@ -45,9 +45,9 @@ fi /bin/echo -e "\n### Changing version in version.py..." sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py -/bin/echo -e "\n### Committing README.md and youtube_dl/version.py..." -make README.md -git add README.md youtube_dl/version.py +/bin/echo -e "\n### Committing documentation and youtube_dl/version.py..." +make README.md CONTRIBUTING.md supportedsites +git add README.md CONTRIBUTING.md docs/supportedsites.md youtube_dl/version.py git commit -m "release $version" /bin/echo -e "\n### Now tagging, signing and pushing..." diff --git a/docs/supportedsites.md b/docs/supportedsites.md new file mode 100644 index 000000000..dbff5e270 --- /dev/null +++ b/docs/supportedsites.md @@ -0,0 +1,565 @@ +# Supported sites + - **1tv**: Первый канал + - **1up.com** + - **220.ro** + - **24video** + - **3sat** + - **4tube** + - **56.com** + - **5min** + - **8tracks** + - **9gag** + - **abc.net.au** + - **Abc7News** + - **AcademicEarth:Course** + - **AddAnime** + - **AdobeTV** + - **AdultSwim** + - **Aftenposten** + - **Aftonbladet** + - **AlJazeera** + - **Allocine** + - **AlphaPorno** + - **anitube.se** + - **AnySex** + - **Aparat** + - **AppleDailyAnimationNews** + - **AppleDailyRealtimeNews** + - **AppleTrailers** + - **archive.org**: archive.org videos + - **ARD** + - **ARD:mediathek** + - **arte.tv** + - **arte.tv:+7** + - **arte.tv:concert** + - **arte.tv:creative** + - **arte.tv:ddc** + - **arte.tv:embed** + - **arte.tv:future** + - **AtresPlayer** + - **ATTTechChannel** + - **audiomack** + - **audiomack:album** + - **Azubu** + - **bambuser** + - **bambuser:channel** + - **Bandcamp** + - **Bandcamp:album** + - **bbc.co.uk**: BBC iPlayer + - **Beeg** + - **BehindKink** + - **Bet** + - **Bild**: Bild.de + - **BiliBili** + - **blinkx** + - **blip.tv:user** + - **BlipTV** + - **Bloomberg** + - **Bpb**: Bundeszentrale für politische Bildung + - **BR**: Bayerischer Rundfunk Mediathek + - **Break** + - **Brightcove** + - **BuzzFeed** + - **BYUtv** + - **Camdemy** + - **CamdemyFolder** + - **Canal13cl** + - **canalc2.tv** + - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv + - **CBS** + - **CBSNews**: CBS News + - **CBSSports** + - **CeskaTelevize** + - **channel9**: Channel 9 + - **Chilloutzone** + - **Cinchcast** + - **Cinemassacre** + - **clipfish** + - **cliphunter** + - **Clipsyndicate** + - **Cloudy** + - **Clubic** + - **cmt.com** + - **CNET** + - **CNN** + - **CNNArticle** + - **CNNBlogs** + - **CollegeHumor** + - **CollegeRama** + - **ComCarCoff** + - **ComedyCentral** + - **ComedyCentralShows**: The Daily Show / The Colbert Report + - **CondeNast**: Condé Nast media group: Condé Nast, GQ, Glamour, Vanity Fair, Vogue, W Magazine, WIRED + - **Cracked** + - **Criterion** + - **Crunchyroll** + - **crunchyroll:playlist** + - **CSpan**: C-SPAN + - **CtsNews** + - **culturebox.francetvinfo.fr** + - **dailymotion** + - **dailymotion:playlist** + - **dailymotion:user** + - **daum.net** + - **DBTV** + - **DctpTv** + - **DeezerPlaylist** + - **defense.gouv.fr** + - **Discovery** + - **divxstage**: DivxStage + - **Dotsub** + - **DRBonanza** + - **Dropbox** + - **DrTuber** + - **DRTV** + - **Dump** + - **dvtv**: http://video.aktualne.cz/ + - **EbaumsWorld** + - **EchoMsk** + - **eHow** + - **Einthusan** + - **eitb.tv** + - **EllenTV** + - **EllenTV:clips** + - **ElPais**: El País + - **Embedly** + - **EMPFlix** + - **Engadget** + - **Eporner** + - **EroProfile** + - **Escapist** + - **EveryonesMixtape** + - **exfm**: ex.fm + - **ExpoTV** + - **ExtremeTube** + - **facebook** + - **faz.net** + - **fc2** + - **fernsehkritik.tv** + - **fernsehkritik.tv:postecke** + - **Firedrive** + - **Firstpost** + - **Flickr** + - **Folketinget**: Folketinget (ft.dk; Danish parliament) + - **Foxgay** + - **FoxNews** + - **france2.fr:generation-quoi** + - **FranceCulture** + - **FranceInter** + - **francetv**: France 2, 3, 4, 5 and Ô + - **francetvinfo.fr** + - **Freesound** + - **freespeech.org** + - **FreeVideo** + - **FunnyOrDie** + - **Gamekings** + - **GameOne** + - **gameone:playlist** + - **GameSpot** + - **GameStar** + - **Gametrailers** + - **GDCVault** + - **generic**: Generic downloader that works on some sites + - **GiantBomb** + - **Giga** + - **Glide**: Glide mobile video messages (glide.me) + - **Globo** + - **GodTube** + - **GoldenMoustache** + - **Golem** + - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in and fastvideo.in + - **Goshgay** + - **Grooveshark** + - **Groupon** + - **Hark** + - **HearThisAt** + - **Heise** + - **HellPorno** + - **Helsinki**: helsinki.fi + - **HentaiStigma** + - **HistoricFilms** + - **History** + - **hitbox** + - **hitbox:live** + - **HornBunny** + - **HostingBulk** + - **HotNewHipHop** + - **Howcast** + - **HowStuffWorks** + - **HuffPost**: Huffington Post + - **Hypem** + - **Iconosquare** + - **ign.com** + - **imdb**: Internet Movie Database trailers + - **imdb:list**: Internet Movie Database lists + - **Imgur** + - **Ina** + - **InfoQ** + - **Instagram** + - **instagram:user**: Instagram user profile + - **InternetVideoArchive** + - **IPrima** + - **ivi**: ivi.ru + - **ivi:compilation**: ivi.ru compilations + - **Izlesene** + - **JadoreCettePub** + - **JeuxVideo** + - **Jove** + - **jpopsuki.tv** + - **Jukebox** + - **Kankan** + - **Karaoketv** + - **keek** + - **KeezMovies** + - **KhanAcademy** + - **KickStarter** + - **kontrtube**: KontrTube.ru - Труба зовёт + - **KrasView**: Красвью + - **Ku6** + - **la7.tv** + - **Laola1Tv** + - **lifenews**: LIFE | NEWS + - **LiveLeak** + - **livestream** + - **livestream:original** + - **LnkGo** + - **lrt.lt** + - **lynda**: lynda.com videos + - **lynda:course**: lynda.com online courses + - **m6** + - **macgamestore**: MacGameStore trailers + - **mailru**: Видео@Mail.Ru + - **Malemotion** + - **MDR** + - **media.ccc.de** + - **metacafe** + - **Metacritic** + - **Mgoon** + - **Minhateca** + - **MinistryGrid** + - **mitele.es** + - **mixcloud** + - **MLB** + - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net + - **Mofosex** + - **Mojvideo** + - **Moniker**: allmyvideos.net and vidspot.net + - **mooshare**: Mooshare.biz + - **Morningstar**: morningstar.com + - **Motherless** + - **Motorsport**: motorsport.com + - **MovieClips** + - **Moviezine** + - **movshare**: MovShare + - **MPORA** + - **MTV** + - **mtviggy.com** + - **mtvservices:embedded** + - **MuenchenTV**: münchen.tv + - **MusicPlayOn** + - **MusicVault** + - **muzu.tv** + - **MySpace** + - **MySpace:album** + - **MySpass** + - **myvideo** + - **MyVidster** + - **n-tv.de** + - **NationalGeographic** + - **Naver** + - **NBA** + - **NBC** + - **NBCNews** + - **ndr**: NDR.de - Mediathek + - **NDTV** + - **NerdCubedFeed** + - **Nerdist** + - **Netzkino** + - **Newgrounds** + - **Newstube** + - **NextMedia** + - **NextMediaActionNews** + - **nfb**: National Film Board of Canada + - **nfl.com** + - **nhl.com** + - **nhl.com:news**: NHL news + - **nhl.com:videocenter**: NHL videocenter category + - **niconico**: ニコニコ動画 + - **NiconicoPlaylist** + - **Noco** + - **Normalboots** + - **NosVideo** + - **novamov**: NovaMov + - **Nowness** + - **nowvideo**: NowVideo + - **npo.nl** + - **npo.nl:live** + - **npo.nl:radio** + - **npo.nl:radio:fragment** + - **NRK** + - **NRKTV** + - **ntv.ru** + - **Nuvid** + - **NYTimes** + - **ocw.mit.edu** + - **OktoberfestTV** + - **on.aol.com** + - **Ooyala** + - **OpenFilm** + - **orf:fm4**: radio FM4 + - **orf:oe1**: Radio Österreich 1 + - **orf:tvthek**: ORF TVthek + - **parliamentlive.tv**: UK parliament videos + - **Patreon** + - **PBS** + - **Phoenix** + - **Photobucket** + - **PlanetaPlay** + - **play.fm** + - **played.to** + - **Playvid** + - **plus.google**: Google Plus + - **pluzz.francetv.fr** + - **podomatic** + - **PornHd** + - **PornHub** + - **PornHubPlaylist** + - **Pornotube** + - **PornoXO** + - **PromptFile** + - **prosiebensat1**: ProSiebenSat.1 Digital + - **Pyvideo** + - **QuickVid** + - **radio.de** + - **radiobremen** + - **radiofrance** + - **Rai** + - **RBMARadio** + - **RedTube** + - **Restudy** + - **ReverbNation** + - **RingTV** + - **RottenTomatoes** + - **Roxwel** + - **RTBF** + - **Rte** + - **rtl.nl**: rtl.nl and rtlxl.nl + - **RTL2** + - **RTLnow** + - **RTP** + - **RTS**: RTS.ch + - **rtve.es:alacarta**: RTVE a la carta + - **rtve.es:live**: RTVE.es live streams + - **RUHD** + - **rutube**: Rutube videos + - **rutube:channel**: Rutube channels + - **rutube:embed**: Rutube embedded videos + - **rutube:movie**: Rutube movies + - **rutube:person**: Rutube person videos + - **RUTV**: RUTV.RU + - **Sandia**: Sandia National Laboratories + - **Sapo**: SAPO Vídeos + - **savefrom.net** + - **SBS**: sbs.com.au + - **SciVee** + - **screen.yahoo:search**: Yahoo screen search + - **Screencast** + - **ScreencastOMatic** + - **ScreenwaveMedia** + - **ServingSys** + - **Sexu** + - **SexyKarma**: Sexy Karma and Watch Indian Porn + - **Shared** + - **ShareSix** + - **Sina** + - **Slideshare** + - **Slutload** + - **smotri**: Smotri.com + - **smotri:broadcast**: Smotri.com broadcasts + - **smotri:community**: Smotri.com community videos + - **smotri:user**: Smotri.com user videos + - **Snotr** + - **Sockshare** + - **Sohu** + - **soundcloud** + - **soundcloud:playlist** + - **soundcloud:set** + - **soundcloud:user** + - **Soundgasm** + - **southpark.cc.com** + - **southpark.de** + - **Space** + - **Spankwire** + - **Spiegel** + - **Spiegel:Article**: Articles on spiegel.de + - **Spiegeltv** + - **Spike** + - **Sport5** + - **SportBox** + - **SportDeutschland** + - **SRMediathek**: Saarländischer Rundfunk + - **stanfordoc**: Stanford Open ClassRoom + - **Steam** + - **streamcloud.eu** + - **StreamCZ** + - **StreetVoice** + - **SunPorno** + - **SVTPlay** + - **SWRMediathek** + - **Syfy** + - **SztvHu** + - **Tagesschau** + - **Tapely** + - **Tass** + - **teachertube**: teachertube.com videos + - **teachertube:user:collection**: teachertube.com user and collection videos + - **TeachingChannel** + - **Teamcoco** + - **TeamFour** + - **TechTalks** + - **techtv.mit.edu** + - **TED** + - **tegenlicht.vpro.nl** + - **TeleBruxelles** + - **telecinco.es** + - **TeleMB** + - **TeleTask** + - **TenPlay** + - **TestTube** + - **TF1** + - **TheOnion** + - **ThePlatform** + - **TheSixtyOne** + - **ThisAV** + - **THVideo** + - **THVideoPlaylist** + - **tinypic**: tinypic.com videos + - **tlc.com** + - **tlc.de** + - **TMZ** + - **TNAFlix** + - **tou.tv** + - **Toypics**: Toypics user profile + - **ToypicsUser**: Toypics user profile + - **TrailerAddict** (Currently broken) + - **Trilulilu** + - **TruTube** + - **Tube8** + - **Tudou** + - **Tumblr** + - **TuneIn** + - **Turbo** + - **Tutv** + - **tv.dfb.de** + - **TV4**: tv4.se and tv4play.se + - **tvigle**: Интернет-телевидение Tvigle.ru + - **tvp.pl** + - **tvp.pl:Series** + - **TVPlay**: TV3Play and related services + - **Tweakers** + - **twitch:bookmarks** + - **twitch:chapter** + - **twitch:past_broadcasts** + - **twitch:profile** + - **twitch:stream** + - **twitch:video** + - **twitch:vod** + - **Ubu** + - **udemy** + - **udemy:course** + - **Unistra** + - **Urort**: NRK P3 Urørt + - **ustream** + - **ustream:channel** + - **Vbox7** + - **VeeHD** + - **Veoh** + - **Vesti**: Вести.Ru + - **Vevo** + - **VGTV** + - **vh1.com** + - **Vice** + - **Viddler** + - **video.google:search**: Google Video search + - **video.mit.edu** + - **VideoBam** + - **VideoDetective** + - **videofy.me** + - **videolectures.net** + - **VideoMega** + - **VideoPremium** + - **VideoTt**: video.tt - Your True Tube + - **videoweed**: VideoWeed + - **Vidme** + - **Vidzi** + - **vier** + - **vier:videos** + - **viki** + - **vimeo** + - **vimeo:album** + - **vimeo:channel** + - **vimeo:group** + - **vimeo:likes**: Vimeo user likes + - **vimeo:review**: Review pages on vimeo + - **vimeo:user** + - **vimeo:watchlater**: Vimeo watch later list, "vimeowatchlater" keyword (requires authentication) + - **Vimple**: Vimple.ru + - **Vine** + - **vine:user** + - **vk.com** + - **vk.com:user-videos**: vk.com:All of a user's videos + - **Vodlocker** + - **Vporn** + - **VRT** + - **vube**: Vube.com + - **VuClip** + - **vulture.com** + - **Walla** + - **WashingtonPost** + - **wat.tv** + - **WayOfTheMaster** + - **WDR** + - **wdr:mobile** + - **WDRMaus**: Sendung mit der Maus + - **WebOfStories** + - **Weibo** + - **Wimp** + - **Wistia** + - **WorldStarHipHop** + - **wrzuta.pl** + - **WSJ**: Wall Street Journal + - **XBef** + - **XboxClips** + - **XHamster** + - **XMinus** + - **XNXX** + - **XTube** + - **XTubeUser**: XTube user profile + - **Xuite** + - **XVideos** + - **XXXYMovies** + - **Yahoo**: Yahoo screen and movies + - **Yam** + - **YesJapan** + - **Ynet** + - **YouJizz** + - **Youku** + - **YouPorn** + - **YourUpload** + - **youtube**: YouTube.com + - **youtube:channel**: YouTube.com channels + - **youtube:favorites**: YouTube.com favourite videos, ":ytfav" for short (requires authentication) + - **youtube:history**: Youtube watch history, ":ythistory" for short (requires authentication) + - **youtube:playlist**: YouTube.com playlists + - **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication) + - **youtube:search**: YouTube.com searches + - **youtube:search:date**: YouTube.com searches, newest videos first + - **youtube:search_url**: YouTube.com search URLs + - **youtube:show**: YouTube.com (multi-season) shows + - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication) + - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword) + - **youtube:watch_later**: Youtube watch later list, ":ytwatchlater" for short (requires authentication) + - **ZDF** + - **ZDFChannel** + - **zingmp3:album**: mp3.zing.vn albums + - **zingmp3:song**: mp3.zing.vn songs diff --git a/setup.cfg b/setup.cfg index e57d130e3..26857750c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,6 @@ [wheel] universal = True + +[flake8] +exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build,.git +ignore = E402,E501,E731 diff --git a/test/helper.py b/test/helper.py index 8af57ead5..bd7d374f9 100644 --- a/test/helper.py +++ b/test/helper.py @@ -82,24 +82,14 @@ class FakeYDL(YoutubeDL): def gettestcases(include_onlymatching=False): for ie in youtube_dl.extractor.gen_extractors(): - t = getattr(ie, '_TEST', None) - if t: - assert not hasattr(ie, '_TESTS'), \ - '%s has _TEST and _TESTS' % type(ie).__name__ - tests = [t] - else: - tests = getattr(ie, '_TESTS', []) - for t in tests: - if not include_onlymatching and t.get('only_matching', False): - continue - t['name'] = type(ie).__name__[:-len('IE')] - yield t + for tc in ie.get_testcases(include_onlymatching): + yield tc md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest() -def expect_info_dict(self, expected_dict, got_dict): +def expect_info_dict(self, got_dict, expected_dict): for info_field, expected in expected_dict.items(): if isinstance(expected, compat_str) and expected.startswith('re:'): got = got_dict.get(info_field) @@ -113,6 +103,26 @@ def expect_info_dict(self, expected_dict, got_dict): self.assertTrue( match_rex.match(got), 'field %s (value: %r) should match %r' % (info_field, got, match_str)) + elif isinstance(expected, compat_str) and expected.startswith('startswith:'): + got = got_dict.get(info_field) + start_str = expected[len('startswith:'):] + self.assertTrue( + isinstance(got, compat_str), + 'Expected a %s object, but got %s for field %s' % ( + compat_str.__name__, type(got).__name__, info_field)) + self.assertTrue( + got.startswith(start_str), + 'field %s (value: %r) should start with %r' % (info_field, got, start_str)) + elif isinstance(expected, compat_str) and expected.startswith('contains:'): + got = got_dict.get(info_field) + contains_str = expected[len('contains:'):] + self.assertTrue( + isinstance(got, compat_str), + 'Expected a %s object, but got %s for field %s' % ( + compat_str.__name__, type(got).__name__, info_field)) + self.assertTrue( + contains_str in got, + 'field %s (value: %r) should contain %r' % (info_field, got, contains_str)) elif isinstance(expected, type): got = got_dict.get(info_field) self.assertTrue(isinstance(got, expected), @@ -120,6 +130,20 @@ def expect_info_dict(self, expected_dict, got_dict): else: if isinstance(expected, compat_str) and expected.startswith('md5:'): got = 'md5:' + md5(got_dict.get(info_field)) + elif isinstance(expected, compat_str) and expected.startswith('mincount:'): + got = got_dict.get(info_field) + self.assertTrue( + isinstance(got, list), + 'Expected field %s to be a list, but it is of type %s' % ( + info_field, type(got).__name__)) + expected_num = int(expected.partition(':')[2]) + assertGreaterEqual( + self, len(got), expected_num, + 'Expected %d items in field %s, but only got %d' % ( + expected_num, info_field, len(got) + ) + ) + continue else: got = got_dict.get(info_field) self.assertEqual(expected, got, @@ -137,7 +161,7 @@ def expect_info_dict(self, expected_dict, got_dict): # Are checkable fields missing from the test case definition? test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value)) for key, value in got_dict.items() - if value and key in ('title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location')) + if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location')) missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys()) if missing_keys: def _repr(v): @@ -145,11 +169,19 @@ def expect_info_dict(self, expected_dict, got_dict): return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'").replace('\n', '\\n') else: return repr(v) - info_dict_str = ''.join( - ' %s: %s,\n' % (_repr(k), _repr(v)) - for k, v in test_info_dict.items()) + info_dict_str = '' + if len(missing_keys) != len(expected_dict): + info_dict_str += ''.join( + ' %s: %s,\n' % (_repr(k), _repr(v)) + for k, v in test_info_dict.items() if k not in missing_keys) + + if info_dict_str: + info_dict_str += '\n' + info_dict_str += ''.join( + ' %s: %s,\n' % (_repr(k), _repr(test_info_dict[k])) + for k in missing_keys) write_string( - '\n\'info_dict\': {\n' + info_dict_str + '}\n', out=sys.stderr) + '\n\'info_dict\': {\n' + info_dict_str + '},\n', out=sys.stderr) self.assertFalse( missing_keys, 'Missing keys in test definition: %s' % ( @@ -162,7 +194,9 @@ def assertRegexpMatches(self, text, regexp, msg=None): else: m = re.match(regexp, text) if not m: - note = 'Regexp didn\'t match: %r not found in %r' % (regexp, text) + note = 'Regexp didn\'t match: %r not found' % (regexp) + if len(text) < 1000: + note += ' in %r' % text if msg is None: msg = note else: diff --git a/test/parameters.json b/test/parameters.json index 098cd0cd0..af77b89b4 100644 --- a/test/parameters.json +++ b/test/parameters.json @@ -39,5 +39,6 @@ "writesubtitles": false, "allsubtitles": false, "listssubtitles": false, - "socket_timeout": 20 + "socket_timeout": 20, + "fixup": "never" } diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py index 13c18ed95..be8d12997 100644 --- a/test/test_InfoExtractor.py +++ b/test/test_InfoExtractor.py @@ -40,5 +40,23 @@ class TestInfoExtractor(unittest.TestCase): self.assertEqual(ie._og_search_description(html), 'Some video\'s description ') self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2') + def test_html_search_meta(self): + ie = self.ie + html = ''' + + + + + + + ''' + + self.assertEqual(ie._html_search_meta('a', html), '1') + self.assertEqual(ie._html_search_meta('b', html), '2') + self.assertEqual(ie._html_search_meta('c', html), '3') + self.assertEqual(ie._html_search_meta('d', html), '4') + self.assertEqual(ie._html_search_meta('e', html), '5') + self.assertEqual(ie._html_search_meta('f', html), '6') + if __name__ == '__main__': unittest.main() diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index f8e4f930e..b1cd6a69f 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -8,9 +8,12 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import copy + from test.helper import FakeYDL, assertRegexpMatches from youtube_dl import YoutubeDL from youtube_dl.extractor import YoutubeIE +from youtube_dl.postprocessor.common import PostProcessor class YDL(FakeYDL): @@ -192,6 +195,37 @@ class TestFormatSelection(unittest.TestCase): downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], 'vid-high') + def test_format_selection_audio_exts(self): + formats = [ + {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'}, + {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'}, + {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'}, + {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'}, + {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'}, + ] + + info_dict = _make_result(formats) + ydl = YDL({'format': 'best'}) + ie = YoutubeIE(ydl) + ie._sort_formats(info_dict['formats']) + ydl.process_ie_result(copy.deepcopy(info_dict)) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'aac-64') + + ydl = YDL({'format': 'mp3'}) + ie = YoutubeIE(ydl) + ie._sort_formats(info_dict['formats']) + ydl.process_ie_result(copy.deepcopy(info_dict)) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'mp3-64') + + ydl = YDL({'prefer_free_formats': True}) + ie = YoutubeIE(ydl) + ie._sort_formats(info_dict['formats']) + ydl.process_ie_result(copy.deepcopy(info_dict)) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'ogg-64') + def test_format_selection_video(self): formats = [ {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': '_'}, @@ -218,7 +252,7 @@ class TestFormatSelection(unittest.TestCase): # 3D '85', '84', '102', '83', '101', '82', '100', # Dash video - '138', '137', '248', '136', '247', '135', '246', + '137', '248', '136', '247', '135', '246', '245', '244', '134', '243', '133', '242', '160', # Dash audio '141', '172', '140', '171', '139', @@ -248,6 +282,61 @@ class TestFormatSelection(unittest.TestCase): downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], f1id) + def test_format_filtering(self): + formats = [ + {'format_id': 'A', 'filesize': 500, 'width': 1000}, + {'format_id': 'B', 'filesize': 1000, 'width': 500}, + {'format_id': 'C', 'filesize': 1000, 'width': 400}, + {'format_id': 'D', 'filesize': 2000, 'width': 600}, + {'format_id': 'E', 'filesize': 3000}, + {'format_id': 'F'}, + {'format_id': 'G', 'filesize': 1000000}, + ] + for f in formats: + f['url'] = 'http://_/' + f['ext'] = 'unknown' + info_dict = _make_result(formats) + + ydl = YDL({'format': 'best[filesize<3000]'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'D') + + ydl = YDL({'format': 'best[filesize<=3000]'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'E') + + ydl = YDL({'format': 'best[filesize <= ? 3000]'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'F') + + ydl = YDL({'format': 'best [filesize = 1000] [width>450]'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'B') + + ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'C') + + ydl = YDL({'format': '[filesize>?1]'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'G') + + ydl = YDL({'format': '[filesize<1M]'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'E') + + ydl = YDL({'format': '[filesize<1MiB]'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], 'G') + def test_add_extra_info(self): test_dict = { 'extractor': 'Foo', @@ -282,5 +371,35 @@ class TestFormatSelection(unittest.TestCase): 'vbr': 10, }), '^\s*10k$') + def test_postprocessors(self): + filename = 'post-processor-testfile.mp4' + audiofile = filename + '.mp3' + + class SimplePP(PostProcessor): + def run(self, info): + with open(audiofile, 'wt') as f: + f.write('EXAMPLE') + info['filepath'] + return False, info + + def run_pp(params): + with open(filename, 'wt') as f: + f.write('EXAMPLE') + ydl = YoutubeDL(params) + ydl.add_post_processor(SimplePP()) + ydl.post_process(filename, {'filepath': filename}) + + run_pp({'keepvideo': True}) + self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename) + self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile) + os.unlink(filename) + os.unlink(audiofile) + + run_pp({'keepvideo': False}) + self.assertFalse(os.path.exists(filename), '%s exists' % filename) + self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile) + os.unlink(audiofile) + + if __name__ == '__main__': unittest.main() diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py index 5be065c43..6f5513faa 100644 --- a/test/test_age_restriction.py +++ b/test/test_age_restriction.py @@ -45,11 +45,6 @@ class TestAgeRestriction(unittest.TestCase): 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/', '505835.mp4', 2, old_age=25) - def test_pornotube(self): - self._assert_restricted( - 'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing', - '1689755.flv', 13) - if __name__ == '__main__': unittest.main() diff --git a/test/test_all_urls.py b/test/test_all_urls.py index bd4fe17bf..e66264b4b 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -14,7 +14,6 @@ from test.helper import gettestcases from youtube_dl.extractor import ( FacebookIE, gen_extractors, - TwitchIE, YoutubeIE, ) @@ -72,18 +71,6 @@ class TestAllURLsMatching(unittest.TestCase): self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url']) self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url']) - def test_twitch_channelid_matching(self): - self.assertTrue(TwitchIE.suitable('twitch.tv/vanillatv')) - self.assertTrue(TwitchIE.suitable('www.twitch.tv/vanillatv')) - self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv')) - self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/')) - - def test_twitch_videoid_matching(self): - self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/b/328087483')) - - def test_twitch_chapterid_matching(self): - self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361')) - def test_youtube_extract(self): assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id) assertExtractId('http://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc') @@ -115,8 +102,6 @@ class TestAllURLsMatching(unittest.TestCase): self.assertMatch(':ythistory', ['youtube:history']) self.assertMatch(':thedailyshow', ['ComedyCentralShows']) self.assertMatch(':tds', ['ComedyCentralShows']) - self.assertMatch(':colbertreport', ['ComedyCentralShows']) - self.assertMatch(':cr', ['ComedyCentralShows']) def test_vimeo_matching(self): self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel']) diff --git a/test/test_download.py b/test/test_download.py index a009aa475..6a149ae4f 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -89,7 +89,7 @@ def generator(test_case): for tc in test_cases: info_dict = tc.get('info_dict', {}) - if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')): + if not (info_dict.get('id') and info_dict.get('ext')): raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?') if 'skip' in test_case: @@ -116,7 +116,7 @@ def generator(test_case): expect_warnings(ydl, test_case.get('expected_warnings', [])) def get_tc_filename(tc): - return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {})) + return ydl.prepare_filename(tc.get('info_dict', {})) res_dict = None @@ -155,7 +155,7 @@ def generator(test_case): if is_playlist: self.assertEqual(res_dict['_type'], 'playlist') self.assertTrue('entries' in res_dict) - expect_info_dict(self, test_case.get('info_dict', {}), res_dict) + expect_info_dict(self, res_dict, test_case.get('info_dict', {})) if 'playlist_mincount' in test_case: assertGreaterEqual( @@ -204,7 +204,7 @@ def generator(test_case): with io.open(info_json_fn, encoding='utf-8') as infof: info_dict = json.load(infof) - expect_info_dict(self, tc.get('info_dict', {}), info_dict) + expect_info_dict(self, info_dict, tc.get('info_dict', {})) finally: try_rm_tcs_files() if is_playlist and res_dict is not None and res_dict.get('entries'): diff --git a/test/test_http.py b/test/test_http.py new file mode 100644 index 000000000..bd4d46fef --- /dev/null +++ b/test/test_http.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +# Allow direct execution +import os +import sys +import unittest +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from youtube_dl import YoutubeDL +from youtube_dl.compat import compat_http_server +import ssl +import threading + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass + + def do_GET(self): + if self.path == '/video.html': + self.send_response(200) + self.send_header('Content-Type', 'text/html; charset=utf-8') + self.end_headers() + self.wfile.write(b'