From 7b117c3c3449cb3a7e04a51c629597535e218a22 Mon Sep 17 00:00:00 2001 From: Janez Troha Date: Tue, 22 Feb 2011 14:28:48 +0100 Subject: [PATCH 1/4] Added transcoder based on ffmpeg --- LATEST_VERSION | 2 +- youtube-dl | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/LATEST_VERSION b/LATEST_VERSION index 4ab209346..1b1e085d8 100644 --- a/LATEST_VERSION +++ b/LATEST_VERSION @@ -1 +1 @@ -2011.01.30 +2011.02.22 diff --git a/youtube-dl b/youtube-dl index 0dd872e50..cf46b1f4d 100755 --- a/youtube-dl +++ b/youtube-dl @@ -7,6 +7,7 @@ # Author: Witold Baryluk # Author: Paweł Paprota # Author: Gergely Imreh +# Author: Janez Troha # License: Public domain code import cookielib import ctypes @@ -21,6 +22,7 @@ import netrc import os import os.path import re +import shlex import socket import string import StringIO @@ -2609,6 +2611,44 @@ class PostProcessor(object): """ return information # by default, do nothing +class Transcode(PostProcessor): + """Post Processor for file transcoding""" + def __init__(self,file_type, args): + # Check for ffmepg first + try: + subprocess.call(['ffmpeg', '-v'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT) + except (OSError, IOError): + raise PostProcessingError(u'ERROR: "ffmpeg" could not be found') + + self.file_type = file_type + + if args: + self.args = str(args) + else: + self.args = str("") + + PostProcessor.__init__(self, None) + + def run(self, information): + self._file_path = str(information["filepath"]) + basename, extension = os.path.splitext(self._file_path) + self._new_file_path = self._file_path.replace(extension, "."+self.file_type) + self._downloader.to_screen(u'[transcode] Started transcoding of %s to %s' % ( self._file_path, self._new_file_path) ) + stringish_command = 'ffmpeg -y -i "%s" %s "%s"' % (self._file_path, self.args, self._new_file_path) + self.encode(stringish_command) + return None + + def encode(self, cmd): + pipe = subprocess.Popen( + shlex.split( cmd ), + stdout=(file(os.path.devnull, 'w')), + stderr=subprocess.STDOUT + ) + retval = pipe.wait() + while retval == 2 or retval == 1: + pass + self._downloader.to_screen(u'[transcode] Completed transcoding of %s to %s' % ( self._file_path, self._new_file_path) ) + ### MAIN PROGRAM ### if __name__ == '__main__': try: @@ -2733,6 +2773,13 @@ if __name__ == '__main__': help='do not use the Last-modified header to set the file modification time', default=True) parser.add_option_group(filesystem) + transcode = optparse.OptionGroup(parser, 'Transcoding Options (uses ffmpeg)') + transcode.add_option('-T','--transcode_to', + action='store', dest='transcode_to', metavar='FILE_TYPE', help='transcode to specific video or audio format (example: mp3 mp4 mov mkv)', default=False) + transcode.add_option('--transcode_extra', + action='store', dest='transcode_extra', metavar='ARGS', help='pass additional parameters to ffmpeg (example: -vcodec libx264 -vpre slow -vpre ipod640 -b 2048k -acodec libfaac -ab 96k)', default=False) + parser.add_option_group(transcode) + (opts, args) = parser.parse_args() # Open appropriate CookieJar @@ -2876,6 +2923,14 @@ if __name__ == '__main__': # fallback if none of the others work fd.add_info_extractor(generic_ie) + # Transcodig parser + if opts.transcode_to: + try: + transcoder = Transcode(opts.transcode_to, opts.transcode_extra) + fd.add_post_processor(transcoder) + except PostProcessingError, err: + sys.exit(err) + # Update version if opts.update_self: update_self(fd, sys.argv[0]) From 19a59ed26f22f87feb0a663c05b271df43889bc2 Mon Sep 17 00:00:00 2001 From: Janez Troha Date: Tue, 22 Feb 2011 15:28:51 +0100 Subject: [PATCH 2/4] Readme and javascript generic search --- readme.md | 102 +++++++++++++++++++++++++++++++++++++++++++++++++++++ youtube-dl | 7 ++-- 2 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 readme.md diff --git a/readme.md b/readme.md new file mode 100644 index 000000000..a095a5e53 --- /dev/null +++ b/readme.md @@ -0,0 +1,102 @@ +# youtube-dl + +Allows you to download and transcode video from supported sites. Needs ffmpeg for transcoding to work. + +## Supported sites + + * Youtube + * Metacafe + * Dailymotion + * Google + * Photobucket + * Yahoo + * DepositFiles + * Facebook + * Sites with JW Player +## Installation + + $ sudo apt-get install ffmpeg # for Ubuntu + $ sudo wget --no-check-certificate https://github.com/dz0ny/youtube-dl/raw/master/youtube-dl -O /usr/local/bin/youtube-dl && sudo chmod a+x /usr/local/bin/youtube-dl + +## Usage + $ youtube-dl [options] url... + + Options: + -h, --help print this help text and exit + -v, --version print program version and exit + -U, --update update this program to latest stable version + -i, --ignore-errors continue on download errors + -r LIMIT, --rate-limit=LIMIT + download rate limit (e.g. 50k or 44.6m) + -R RETRIES, --retries=RETRIES + number of retries (default is 10) + --playlist-start=NUMBER + playlist video to start at (default is 1) + --playlist-end=NUMBER + playlist video to end at (default is last) + --dump-user-agent display the current browser identification + + Authentication Options: + -u USERNAME, --username=USERNAME + account username + -p PASSWORD, --password=PASSWORD + account password + -n, --netrc use .netrc authentication data + + Video Format Options: + -f FORMAT, --format=FORMAT + video format code + --all-formats download all available video formats + --max-quality=FORMAT + highest quality format to download + + Verbosity / Simulation Options: + -q, --quiet activates quiet mode + -s, --simulate do not download video + -g, --get-url simulate, quiet but print URL + -e, --get-title simulate, quiet but print title + --get-thumbnail simulate, quiet but print thumbnail URL + --get-description simulate, quiet but print video description + --get-filename simulate, quiet but print output filename + --no-progress do not print progress bar + --console-title display progress in console titlebar + + Filesystem Options: + -t, --title use title in file name + -l, --literal use literal title in file name + -A, --auto-number number downloaded files starting from 00000 + -o TEMPLATE, --output=TEMPLATE + output filename template + -a FILE, --batch-file=FILE + file containing URLs to download ('-' for stdin) + -w, --no-overwrites + do not overwrite files + -c, --continue resume partially downloaded files + --cookies=FILE file to dump cookie jar to + --no-part do not use .part files + --no-mtime do not use the Last-modified header to set the file + modification time + + Transcoding Options (uses ffmpeg): + -T FILE_TYPE, --transcode_to=FILE_TYPE + transcode to specific video or audio format (example: + mp3 mp4 mov mkv) + --transcode_extra=ARGS + pass additional parameters to ffmpeg (example: -vcodec + libx264 -vpre slow -vpre ipod640 -b 2048k -acodec + libfaac -ab 96k) + +# License + + Public domain code + +# Authors + + * Ricardo Garcia Gonzalez + * Danny Colligan + * Benjamin Johnson + * Vasyl' Vavrychuk + * Witold Baryluk + * Paweł Paprota + * Gergely Imreh + * Janez Troha diff --git a/youtube-dl b/youtube-dl index cf46b1f4d..6d29452e7 100755 --- a/youtube-dl +++ b/youtube-dl @@ -1767,7 +1767,10 @@ class GenericIE(InfoExtractor): mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) if mobj is None: # Broaden the search a little bit - mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage) + mobj = re.search(r'[^A-Za-z0-9]?(?:file|source|video_url)=(http[^\'"&]*)', webpage) + if mobj is None: + # Search in javascript + mobj = re.search(r'video_url.+(http[^\'"&]*)', webpage) if mobj is None: self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) return @@ -2930,7 +2933,7 @@ if __name__ == '__main__': fd.add_post_processor(transcoder) except PostProcessingError, err: sys.exit(err) - + # Update version if opts.update_self: update_self(fd, sys.argv[0]) From cc6640b59a68b79979e55e1cca2d578f7d700cab Mon Sep 17 00:00:00 2001 From: Janez Troha Date: Tue, 22 Feb 2011 15:31:07 +0100 Subject: [PATCH 3/4] Fix some styling in readme.md --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index a095a5e53..4007ebd95 100644 --- a/readme.md +++ b/readme.md @@ -13,6 +13,7 @@ Allows you to download and transcode video from supported sites. Needs ffmpeg fo * DepositFiles * Facebook * Sites with JW Player + ## Installation $ sudo apt-get install ffmpeg # for Ubuntu From d06017c8d4c8a4abaf054ff88f626dd513974e67 Mon Sep 17 00:00:00 2001 From: Janez Troha Date: Tue, 22 Feb 2011 19:21:30 +0100 Subject: [PATCH 4/4] Added simple post download command processor --- readme.md | 13 +++++++++++++ youtube-dl | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 4007ebd95..59434f2a5 100644 --- a/readme.md +++ b/readme.md @@ -20,6 +20,13 @@ Allows you to download and transcode video from supported sites. Needs ffmpeg fo $ sudo wget --no-check-certificate https://github.com/dz0ny/youtube-dl/raw/master/youtube-dl -O /usr/local/bin/youtube-dl && sudo chmod a+x /usr/local/bin/youtube-dl ## Usage + +### Example + + $ youtube-dl http://www.youtube.com/watch?v=BRHRssf3B6o -T mp4 -C 'sh -c "cp .mp4 && rm "' + + Command line options: + $ youtube-dl [options] url... Options: @@ -86,11 +93,17 @@ Allows you to download and transcode video from supported sites. Needs ffmpeg fo pass additional parameters to ffmpeg (example: -vcodec libx264 -vpre slow -vpre ipod640 -b 2048k -acodec libfaac -ab 96k) + + Post download action: + -C COMMAND, --command=COMMAND + command to run after file has been downloaded + (example: 'sh -c "cp .mp4 && rm "' ) # License Public domain code + # Authors * Ricardo Garcia Gonzalez diff --git a/youtube-dl b/youtube-dl index 6d29452e7..8b1aa9a25 100755 --- a/youtube-dl +++ b/youtube-dl @@ -2639,7 +2639,7 @@ class Transcode(PostProcessor): self._downloader.to_screen(u'[transcode] Started transcoding of %s to %s' % ( self._file_path, self._new_file_path) ) stringish_command = 'ffmpeg -y -i "%s" %s "%s"' % (self._file_path, self.args, self._new_file_path) self.encode(stringish_command) - return None + return information def encode(self, cmd): pipe = subprocess.Popen( @@ -2652,6 +2652,27 @@ class Transcode(PostProcessor): pass self._downloader.to_screen(u'[transcode] Completed transcoding of %s to %s' % ( self._file_path, self._new_file_path) ) +class SimplePostProcessor(PostProcessor): + """Post Processor for file transcoding""" + def __init__(self,command): + self.command_raw = command + PostProcessor.__init__(self, None) + + def run(self, information): + for key, value in information.iteritems(): + self.command_raw = self.command_raw.replace("<" + key + ">", value) + self.command = shlex.split( str( self.command_raw ) ) + self._downloader.to_screen(u'[PostProcessor] Started command %s' % self.command_raw ) + pipe = subprocess.Popen( + ( self.command ) + ) + retval = pipe.wait() + while retval == 2 or retval == 1: + pass + self._downloader.to_screen(u'[PostProcessor] Finnished command %s' % self.command_raw ) + + return information + ### MAIN PROGRAM ### if __name__ == '__main__': try: @@ -2783,6 +2804,11 @@ if __name__ == '__main__': action='store', dest='transcode_extra', metavar='ARGS', help='pass additional parameters to ffmpeg (example: -vcodec libx264 -vpre slow -vpre ipod640 -b 2048k -acodec libfaac -ab 96k)', default=False) parser.add_option_group(transcode) + postprocess = optparse.OptionGroup(parser, 'Post download action') + postprocess.add_option('-C','--command', + action='store', dest='post_download_command', metavar='COMMAND', help='command to run after file has been downloaded (example: "sh -c cp /tmp/.tmp && rm <filepath>" )', default=False) + parser.add_option_group(postprocess) + (opts, args) = parser.parse_args() # Open appropriate CookieJar @@ -2934,6 +2960,14 @@ if __name__ == '__main__': except PostProcessingError, err: sys.exit(err) + # Custom command parser + if opts.post_download_command: + try: + post_download_command = SimplePostProcessor(opts.post_download_command) + fd.add_post_processor(post_download_command) + except PostProcessingError, err: + sys.exit(err) + # Update version if opts.update_self: update_self(fd, sys.argv[0])