From 2cd54089513e45f4bd5dc7d1d31adc096973c08a Mon Sep 17 00:00:00 2001 From: Anthony Weems Date: Thu, 1 May 2014 11:26:21 -0500 Subject: [PATCH 01/10] added flag to merge videos from a playlist into one continuous file --- youtube_dl/YoutubeDL.py | 28 ++++++++++++++++++++++++++- youtube_dl/__init__.py | 4 ++++ youtube_dl/postprocessor/__init__.py | 2 ++ youtube_dl/postprocessor/ffmpeg.py | 29 ++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index f3666573a..6edf54713 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -59,7 +59,7 @@ from .utils import ( ) from .extractor import get_info_extractor, gen_extractors from .downloader import get_suitable_downloader -from .postprocessor import FFmpegMergerPP +from .postprocessor import FFmpegMergerPP, FFmpegConcatPP from .version import __version__ @@ -645,6 +645,32 @@ class YoutubeDL(object): extra_info=extra) playlist_results.append(entry_result) ie_result['entries'] = playlist_results + + if download and not(self.params.get('simulate', False)) and \ + self.params.get('concat', False): + downloaded = [] + concat = FFmpegConcatPP(self) + if not concat._get_executable(): + postprocessors = [] + self.report_warning('You have requested multiple ' + 'formats but ffmpeg or avconv are not installed.' + ' The formats won\'t be merged') + else: + postprocessors = [concat] + for f in ie_result['entries']: + new_info = dict(ie_result) + new_info.update(f) + if 'ext' in new_info: ie_result['ext'] = new_info['ext'] + if 'id' in new_info: ie_result['id'] = new_info['id'] + fname = self.prepare_filename(new_info) + downloaded.append(fname) + + filename = self.prepare_filename(ie_result) + + ie_result['__postprocessors'] = postprocessors + ie_result['__files_to_merge'] = downloaded + self.post_process(filename, ie_result) + return ie_result elif result_type == 'compat_list': def _fixup(r): diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 1d8cf9a09..aa2f3bc83 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -500,6 +500,8 @@ def parseOpts(overrideArguments=None): help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)') postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None, help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)') + postproc.add_option('--concat', action='store_true', dest='concat', + help='Attempt to concatenate multiple videos into one file') postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, help='keeps the video file on disk after the post-processing; the video is erased by default') postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False, @@ -518,6 +520,7 @@ def parseOpts(overrideArguments=None): help='Prefer ffmpeg over avconv for running the postprocessors') + parser.add_option_group(general) parser.add_option_group(selection) parser.add_option_group(downloader) @@ -791,6 +794,7 @@ def _real_main(argv=None): 'bidi_workaround': opts.bidi_workaround, 'debug_printtraffic': opts.debug_printtraffic, 'prefer_ffmpeg': opts.prefer_ffmpeg, + 'concat': opts.concat, 'include_ads': opts.include_ads, 'default_search': opts.default_search, 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest, diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py index 08e6ddd00..d49f5ae3b 100644 --- a/youtube_dl/postprocessor/__init__.py +++ b/youtube_dl/postprocessor/__init__.py @@ -1,6 +1,7 @@ from .atomicparsley import AtomicParsleyPP from .ffmpeg import ( + FFmpegConcatPP, FFmpegAudioFixPP, FFmpegMergerPP, FFmpegMetadataPP, @@ -12,6 +13,7 @@ from .xattrpp import XAttrMetadataPP __all__ = [ 'AtomicParsleyPP', + 'FFmpegConcatPP', 'FFmpegAudioFixPP', 'FFmpegMergerPP', 'FFmpegMetadataPP', diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 602e370f4..08c9e8f79 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -487,6 +487,35 @@ class FFmpegMergerPP(FFmpegPostProcessor): self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args) return True, info +class FFmpegConcatPP(FFmpegPostProcessor): + def run(self, info): + filename = info['filepath'] + args = ['-f', 'concat', '-i', '-', '-c', 'copy'] + self._downloader.to_screen(u'[ffmpeg] Concatenating files into "%s"' % filename) + + if not self._get_executable(): + raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.') + + cmd = ([self._get_executable(), '-y'] + args + + [encodeFilename(self._ffmpeg_filename_argument(filename), True)]) + files = info['__files_to_merge'] + + if self._downloader.params.get('verbose', False): + self._downloader.to_screen(u'[debug] ffmpeg command line: %s' % shell_quote(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + + files_cmd = u'' + for path in files: + encoded_path = encodeFilename(path, True) + files_cmd += u'file \'%s\'\n' % path + stdout, stderr = p.communicate(input=files_cmd) + if p.returncode != 0: + stderr = stderr.decode('utf-8', 'replace') + msg = stderr.strip().split('\n')[-1] + raise FFmpegPostProcessorError(msg) + + for path in files: + os.remove(encodeFilename(path)) class FFmpegAudioFixPP(FFmpegPostProcessor): def run(self, info): From 90e4b63d4cd0eda0b7173d5bc69e1cb5334db8cd Mon Sep 17 00:00:00 2001 From: Anthony Weems Date: Thu, 1 May 2014 12:39:28 -0500 Subject: [PATCH 02/10] return correct value from postprocessor --- youtube_dl/postprocessor/ffmpeg.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 08c9e8f79..44b6c19f6 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -517,6 +517,8 @@ class FFmpegConcatPP(FFmpegPostProcessor): for path in files: os.remove(encodeFilename(path)) + return True, info + class FFmpegAudioFixPP(FFmpegPostProcessor): def run(self, info): filename = info['filepath'] From 997e236cb9e71069b71e8c87c31894db3cb98320 Mon Sep 17 00:00:00 2001 From: Anthony Weems Date: Fri, 2 May 2014 02:24:18 -0500 Subject: [PATCH 03/10] use byte array for communication --- youtube_dl/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 44b6c19f6..c92a97d99 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -508,7 +508,7 @@ class FFmpegConcatPP(FFmpegPostProcessor): for path in files: encoded_path = encodeFilename(path, True) files_cmd += u'file \'%s\'\n' % path - stdout, stderr = p.communicate(input=files_cmd) + stdout, stderr = p.communicate(input=bytes(files_cmd, 'utf-8')) if p.returncode != 0: stderr = stderr.decode('utf-8', 'replace') msg = stderr.strip().split('\n')[-1] From 8aac00e0858f9c29684323cc8a64622dd7cb90bc Mon Sep 17 00:00:00 2001 From: Anthony Weems Date: Fri, 2 May 2014 02:24:39 -0500 Subject: [PATCH 04/10] use encoded path for shell command --- youtube_dl/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index c92a97d99..00a439a24 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -507,7 +507,7 @@ class FFmpegConcatPP(FFmpegPostProcessor): files_cmd = u'' for path in files: encoded_path = encodeFilename(path, True) - files_cmd += u'file \'%s\'\n' % path + files_cmd += u'file \'%s\'\n' % encoded_path stdout, stderr = p.communicate(input=bytes(files_cmd, 'utf-8')) if p.returncode != 0: stderr = stderr.decode('utf-8', 'replace') From 712464e1e8468fc3a085ec7ee591591a9e07558b Mon Sep 17 00:00:00 2001 From: Anthony Weems Date: Fri, 2 May 2014 03:22:37 -0500 Subject: [PATCH 05/10] migrate everything to run_ffmpeg_multiple_files instead of rebuilding the wheel --- youtube_dl/postprocessor/ffmpeg.py | 42 ++++++++++-------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 00a439a24..5a6033e28 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -40,21 +40,29 @@ class FFmpegPostProcessor(PostProcessor): def _uses_avconv(self): return self._get_executable() == self._exes['avconv'] - def run_ffmpeg_multiple_files(self, input_paths, out_path, opts): + def run_ffmpeg_multiple_files(self, input_paths, out_path, opts, via_stdin=False): if not self._get_executable(): raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.') files_cmd = [] + stdin_cmd = u'' for path in input_paths: - files_cmd.extend(['-i', encodeFilename(path, True)]) + if via_stdin: + stdin_cmd += u"file '%s'\n" % encodeFilename(path, True) + else: + files_cmd.extend(['-i', encodeFilename(path, True)]) + cmd = ([self._get_executable(), '-y'] + files_cmd + opts + [encodeFilename(self._ffmpeg_filename_argument(out_path), True)]) if self._downloader.params.get('verbose', False): self._downloader.to_screen(u'[debug] ffmpeg command line: %s' % shell_quote(cmd)) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + if via_stdin: + stdout, stderr = p.communicate(input=bytes(stdin_cmd, 'utf-8')) + else: + stdout, stderr = p.communicate() if p.returncode != 0: stderr = stderr.decode('utf-8', 'replace') msg = stderr.strip().split('\n')[-1] @@ -492,31 +500,9 @@ class FFmpegConcatPP(FFmpegPostProcessor): filename = info['filepath'] args = ['-f', 'concat', '-i', '-', '-c', 'copy'] self._downloader.to_screen(u'[ffmpeg] Concatenating files into "%s"' % filename) - - if not self._get_executable(): - raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.') - - cmd = ([self._get_executable(), '-y'] + args + - [encodeFilename(self._ffmpeg_filename_argument(filename), True)]) - files = info['__files_to_merge'] - - if self._downloader.params.get('verbose', False): - self._downloader.to_screen(u'[debug] ffmpeg command line: %s' % shell_quote(cmd)) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - - files_cmd = u'' - for path in files: - encoded_path = encodeFilename(path, True) - files_cmd += u'file \'%s\'\n' % encoded_path - stdout, stderr = p.communicate(input=bytes(files_cmd, 'utf-8')) - if p.returncode != 0: - stderr = stderr.decode('utf-8', 'replace') - msg = stderr.strip().split('\n')[-1] - raise FFmpegPostProcessorError(msg) - - for path in files: + self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args, via_stdin=True) + for path in info['__files_to_merge']: os.remove(encodeFilename(path)) - return True, info class FFmpegAudioFixPP(FFmpegPostProcessor): From a416c8a7ec39d4aeaf2ec91a28f1489bab0d091e Mon Sep 17 00:00:00 2001 From: Anthony Weems Date: Fri, 2 May 2014 03:23:11 -0500 Subject: [PATCH 06/10] change logic a bit, make variable names more readable --- youtube_dl/YoutubeDL.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 6edf54713..a9d3dbb9c 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -646,29 +646,27 @@ class YoutubeDL(object): playlist_results.append(entry_result) ie_result['entries'] = playlist_results - if download and not(self.params.get('simulate', False)) and \ - self.params.get('concat', False): - downloaded = [] + if self.params['concat']: concat = FFmpegConcatPP(self) - if not concat._get_executable(): + if concat._get_executable(): + postprocessors = [concat] + else: postprocessors = [] self.report_warning('You have requested multiple ' 'formats but ffmpeg or avconv are not installed.' ' The formats won\'t be merged') - else: - postprocessors = [concat] - for f in ie_result['entries']: + + downloaded = [] + for video in playlist_results: new_info = dict(ie_result) - new_info.update(f) - if 'ext' in new_info: ie_result['ext'] = new_info['ext'] - if 'id' in new_info: ie_result['id'] = new_info['id'] + new_info.update(video) + ie_result['ext'] = video['ext'] fname = self.prepare_filename(new_info) downloaded.append(fname) - filename = self.prepare_filename(ie_result) - ie_result['__postprocessors'] = postprocessors ie_result['__files_to_merge'] = downloaded + self.post_process(filename, ie_result) return ie_result From 2bc50ce0fe305aff76d04b4197e0c17721d51156 Mon Sep 17 00:00:00 2001 From: Anthony Weems Date: Fri, 2 May 2014 03:24:36 -0500 Subject: [PATCH 07/10] removed spurious whitespace --- youtube_dl/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index aa2f3bc83..519359a91 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -520,7 +520,6 @@ def parseOpts(overrideArguments=None): help='Prefer ffmpeg over avconv for running the postprocessors') - parser.add_option_group(general) parser.add_option_group(selection) parser.add_option_group(downloader) From 15818d4e21cc3573c7c17c3ae35554a3dbace019 Mon Sep 17 00:00:00 2001 From: Anthony Weems Date: Fri, 2 May 2014 03:32:04 -0500 Subject: [PATCH 08/10] use origin method of getting params --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index a9d3dbb9c..fc8cecc45 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -646,7 +646,7 @@ class YoutubeDL(object): playlist_results.append(entry_result) ie_result['entries'] = playlist_results - if self.params['concat']: + if self.params.get('concat', False): concat = FFmpegConcatPP(self) if concat._get_executable(): postprocessors = [concat] From 76adb9074ab129fab29485f1dc97dc7276ef4849 Mon Sep 17 00:00:00 2001 From: Anthony Weems Date: Fri, 2 May 2014 03:32:44 -0500 Subject: [PATCH 09/10] take --keep-video option into account Unfortunately, we cannot simply return False from FFmpegConcatPP.run() because the current post_process() function merely deletes the specified filename. For this to work properly, the postprocessor should be able to specify the files it wants to delete alongside the flag to delete them. --- youtube_dl/postprocessor/ffmpeg.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 5a6033e28..9d8c055d7 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -501,8 +501,10 @@ class FFmpegConcatPP(FFmpegPostProcessor): args = ['-f', 'concat', '-i', '-', '-c', 'copy'] self._downloader.to_screen(u'[ffmpeg] Concatenating files into "%s"' % filename) self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args, via_stdin=True) - for path in info['__files_to_merge']: - os.remove(encodeFilename(path)) + if self._downloader.params.get('keepvideo', False) is False: + for path in info['__files_to_merge']: + self._downloader.to_screen('Deleting original file %s (pass -k to keep)' % filename) + os.remove(encodeFilename(path)) return True, info class FFmpegAudioFixPP(FFmpegPostProcessor): From 8a7624df3d359b989d96b0027c6fb713a83ec1fd Mon Sep 17 00:00:00 2001 From: Anthony Weems Date: Fri, 9 May 2014 18:29:31 -0500 Subject: [PATCH 10/10] fixed str to byte pattern (compatible with 2.x and 3) --- youtube_dl/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 9d8c055d7..6096ef668 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -60,7 +60,7 @@ class FFmpegPostProcessor(PostProcessor): self._downloader.to_screen(u'[debug] ffmpeg command line: %s' % shell_quote(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) if via_stdin: - stdout, stderr = p.communicate(input=bytes(stdin_cmd, 'utf-8')) + stdout, stderr = p.communicate(input=stdin_cmd.encode('utf-8')) else: stdout, stderr = p.communicate() if p.returncode != 0: