From d94fc760f73c9c4faf32b46c8741b4f0b0595a32 Mon Sep 17 00:00:00 2001 From: AGSPhoenix Date: Mon, 7 Apr 2014 11:52:15 -0400 Subject: [PATCH 1/6] Add ffmpeg concat postprocessor Code's a bit sloppy, but it seems to work. --- youtube_dl/postprocessor/__init__.py | 2 ++ youtube_dl/postprocessor/ffmpeg.py | 23 +++++++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py index 7f19f717f..c003d7a74 100644 --- a/youtube_dl/postprocessor/__init__.py +++ b/youtube_dl/postprocessor/__init__.py @@ -1,6 +1,7 @@ from .ffmpeg import ( FFmpegMergerPP, + FFmpegConcatPP, FFmpegMetadataPP, FFmpegVideoConvertor, FFmpegExtractAudioPP, @@ -10,6 +11,7 @@ from .xattrpp import XAttrMetadataPP __all__ = [ 'FFmpegMergerPP', + 'FFmpegConcatPP', 'FFmpegMetadataPP', 'FFmpegVideoConvertor', 'FFmpegExtractAudioPP', diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 98b5eccb4..9d8344901 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -40,16 +40,19 @@ class FFmpegPostProcessor(PostProcessor): def _uses_avconv(self): return self._get_executable() == self._exes['avconv'] - def run_ffmpeg_multiple_files(self, input_paths, out_path, opts): + def run_ffmpeg_multiple_files(self, input_paths, out_path, opts, preopts=None): if not self._get_executable(): raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.') + preopt_cmd = [] + if preopts: + preopt_cmd = preopts files_cmd = [] for path in input_paths: files_cmd.extend(['-i', encodeFilename(path, True)]) - cmd = ([self._get_executable(), '-y'] + files_cmd - + opts + - [encodeFilename(self._ffmpeg_filename_argument(out_path), True)]) + cmd = ([self._get_executable(), '-y'] + + preopt_cmd + files_cmd + opts + + [encodeFilename(self._ffmpeg_filename_argument(out_path), True)]) if self._downloader.params.get('verbose', False): self._downloader.to_screen(u'[debug] ffmpeg command line: %s' % shell_quote(cmd)) @@ -484,3 +487,15 @@ class FFmpegMergerPP(FFmpegPostProcessor): self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args) return True, info +class FFmpegConcatPP(FFmpegPostProcessor): + def run(self, info): + filename = info['filepath'] + concatargs = ['-f', 'concat'] + args = ['-c', 'copy'] + self._downloader.to_screen(u'[ffmpeg] Appending files into "%s"' % filename) + with open(u'youtube-dl_ffmpeg_append_list.txt', 'wb') as f: + for file in info['__files_to_merge']: + f.write("file '" + file + "'\n") + self.run_ffmpeg_multiple_files([u'youtube-dl_ffmpeg_append_list.txt'], filename, args, preopts=concatargs) + os.unlink('youtube-dl_ffmpeg_append_list.txt') + return True, info From 892328588a80c30d844bd9f9d0ca3a6f5fe8abc7 Mon Sep 17 00:00:00 2001 From: AGSPhoenix Date: Mon, 7 Apr 2014 18:44:35 -0400 Subject: [PATCH 2/6] Implement initial concat support It may not be pretty, but it friggin' works. Some initial testing with The Daily Show, but I have no idea how it handles other sites. --- youtube_dl/YoutubeDL.py | 22 +++++++++++++++++++--- youtube_dl/__init__.py | 4 ++++ youtube_dl/postprocessor/ffmpeg.py | 8 ++++++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 5794fdbe9..827fdebc6 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -58,7 +58,7 @@ from .utils import ( ) from .extractor import get_info_extractor, gen_extractors from .downloader import get_suitable_downloader -from .postprocessor import FFmpegMergerPP +from .postprocessor import FFmpegMergerPP, FFmpegConcatPP from .version import __version__ @@ -640,6 +640,18 @@ class YoutubeDL(object): extra_info=extra) playlist_results.append(entry_result) ie_result['entries'] = playlist_results + + #Run concat PP + if self.params['concat']: + pp = FFmpegConcatPP(self) + + mergelist = [] + for video in playlist_results: + mergelist.append(video['saved_filename']) + ie_result['__files_to_append'] = mergelist + + pp.run(ie_result) + return ie_result elif result_type == 'compat_list': def _fixup(r): @@ -811,13 +823,16 @@ class YoutubeDL(object): for format in formats_to_download: new_info = dict(info_dict) new_info.update(format) - self.process_info(new_info) + saved_filename = self.process_info(new_info) + #Maintain a list of saved files when processing playlists. Used by the concat PP. + info_dict['saved_filename'] = saved_filename # We update the info dict with the best quality format (backwards compatibility) info_dict.update(formats_to_download[-1]) + return info_dict def process_info(self, info_dict): - """Process a single resolved IE result.""" + """Process a single resolved IE result. Returns the saved filename.""" assert info_dict.get('_type', 'video') == 'video' @@ -1019,6 +1034,7 @@ class YoutubeDL(object): return self.record_download_archive(info_dict) + return filename def download(self, url_list): """Download a given list of URLs.""" diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 7e504b75c..4ca067d89 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -97,6 +97,7 @@ from .postprocessor import ( FFmpegExtractAudioPP, FFmpegEmbedSubtitlePP, XAttrMetadataPP, + FFmpegConcatPP, ) @@ -495,6 +496,8 @@ def parseOpts(overrideArguments=None): help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default') postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5', help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)') + postproc.add_option('--join', action='store_true', dest='concat', default=False, + help='when downloading a playlist of multiple videos, try to join them together end-to-end (EXPERIMENTAL)') postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None, help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)') postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, @@ -790,6 +793,7 @@ def _real_main(argv=None): 'default_search': opts.default_search, 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest, 'encoding': opts.encoding, + 'concat': opts.concat, } with YoutubeDL(ydl_opts) as ydl: diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 9d8344901..2a363894e 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -488,13 +488,17 @@ class FFmpegMergerPP(FFmpegPostProcessor): return True, info class FFmpegConcatPP(FFmpegPostProcessor): + #Concat support requires that the IE return '_type' = 'playlist' + #Otherwise it silently fail def run(self, info): - filename = info['filepath'] + filename = info['title'] + u'.mp4' #What could possibly go wrong? concatargs = ['-f', 'concat'] args = ['-c', 'copy'] self._downloader.to_screen(u'[ffmpeg] Appending files into "%s"' % filename) + #According to the ffmpeg docs this is literally how you're supposed to concat files. + #No method using solely the command line is listed. And I'm like "really?". with open(u'youtube-dl_ffmpeg_append_list.txt', 'wb') as f: - for file in info['__files_to_merge']: + for file in info['__files_to_append']: f.write("file '" + file + "'\n") self.run_ffmpeg_multiple_files([u'youtube-dl_ffmpeg_append_list.txt'], filename, args, preopts=concatargs) os.unlink('youtube-dl_ffmpeg_append_list.txt') From 36ffea251b1bd77966b5cf400c3fd65de5aecf86 Mon Sep 17 00:00:00 2001 From: AGSPhoenix Date: Sat, 19 Apr 2014 00:07:34 -0400 Subject: [PATCH 3/6] Document concat parameter --- youtube_dl/YoutubeDL.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 827fdebc6..f5bb3173d 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -128,6 +128,7 @@ class YoutubeDL(object): listsubtitles: Lists all available subtitles for the video subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt) subtitleslangs: List of languages of the subtitles to download + concat: Join videos in a playlist end-to-end keepvideo: Keep the video file after post-processing daterange: A DateRange object, download only if the upload_date is in the range. skip_download: Skip the actual download of the video file From e523a89d97b0912d50c238b9b424f41ed6581e62 Mon Sep 17 00:00:00 2001 From: AGSPhoenix Date: Sat, 19 Apr 2014 00:10:19 -0400 Subject: [PATCH 4/6] Use unique ffmpeg concat filename --- youtube_dl/postprocessor/ffmpeg.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 2a363894e..f0ad9bb20 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -497,9 +497,9 @@ class FFmpegConcatPP(FFmpegPostProcessor): self._downloader.to_screen(u'[ffmpeg] Appending files into "%s"' % filename) #According to the ffmpeg docs this is literally how you're supposed to concat files. #No method using solely the command line is listed. And I'm like "really?". - with open(u'youtube-dl_ffmpeg_append_list.txt', 'wb') as f: + with open(filename + '.list.txt', 'wb') as f: for file in info['__files_to_append']: f.write("file '" + file + "'\n") - self.run_ffmpeg_multiple_files([u'youtube-dl_ffmpeg_append_list.txt'], filename, args, preopts=concatargs) - os.unlink('youtube-dl_ffmpeg_append_list.txt') + self.run_ffmpeg_multiple_files([filename + '.list.txt'], filename, args, preopts=concatargs) + os.unlink(filename + '.list.txt') return True, info From ce9f19fd2bda9ffe6e51782bb0320df97c9389d9 Mon Sep 17 00:00:00 2001 From: AGSPhoenix Date: Sat, 19 Apr 2014 02:03:47 -0400 Subject: [PATCH 5/6] Determine output extension based on input ffmpeg might be able to handle some cases of differing extensions, so I figure just give it a shot and hope it works. What's the worst that could happen? --- youtube_dl/postprocessor/ffmpeg.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index f0ad9bb20..96189f669 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -491,7 +491,14 @@ class FFmpegConcatPP(FFmpegPostProcessor): #Concat support requires that the IE return '_type' = 'playlist' #Otherwise it silently fail def run(self, info): - filename = info['title'] + u'.mp4' #What could possibly go wrong? + #Determine appropriate output extension + extlist = [] + for input_filename in info['__files_to_append']: + extlist.append(os.path.splitext(input_filename)[-1]) + if len(set(extlist)) != 1: + self._downloader.report_warning('Not all files are in the same format! Joining the files may fail.') + + filename = info['title'] + extlist[0] concatargs = ['-f', 'concat'] args = ['-c', 'copy'] self._downloader.to_screen(u'[ffmpeg] Appending files into "%s"' % filename) From 3defc240da1eb9a096c2ded5436700f920b3c49f Mon Sep 17 00:00:00 2001 From: AGSPhoenix Date: Sat, 19 Apr 2014 05:23:07 -0400 Subject: [PATCH 6/6] Escape single quotes in filename There are probably other special characters that will need to be replaced. --- youtube_dl/postprocessor/ffmpeg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 96189f669..49f8a973c 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -506,6 +506,7 @@ class FFmpegConcatPP(FFmpegPostProcessor): #No method using solely the command line is listed. And I'm like "really?". with open(filename + '.list.txt', 'wb') as f: for file in info['__files_to_append']: + file = file.replace("'", "\\'") f.write("file '" + file + "'\n") self.run_ffmpeg_multiple_files([filename + '.list.txt'], filename, args, preopts=concatargs) os.unlink(filename + '.list.txt')