From 960ef979cd9229e86e0854a9d18e7b9b9b184457 Mon Sep 17 00:00:00 2001 From: joncody Date: Sat, 7 Apr 2018 17:44:07 -0700 Subject: [PATCH 1/9] . --- youtube_dl/postprocessor/execafterdownload.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index 64dabe790..33ddd16db 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -20,7 +20,11 @@ class ExecAfterDownloadPP(PostProcessor): if '{}' not in cmd: cmd += ' {}' + # expose the playlist_index and title variables to exec argument + # youtube-dl -x -o "%(playlist_index)s - %(title)s.%(ext)s" --exec "id3v2 -T {playlist_index} -t {title} {}" PLAYLIST_ID cmd = cmd.replace('{}', compat_shlex_quote(information['filepath'])) + cmd = cmd.replace('{title}', compat_shlex_quote(information['title'])) + cmd = cmd.replace('{playlist_index}', str(information['playlist_index'])) self._downloader.to_screen('[exec] Executing command: %s' % cmd) retCode = subprocess.call(encodeArgument(cmd), shell=True) From ac511509bfd22b0eb8be076951711a58d60f3ddd Mon Sep 17 00:00:00 2001 From: joncody Date: Sat, 7 Apr 2018 19:00:39 -0700 Subject: [PATCH 2/9] expose info to exec arg --- youtube_dl/postprocessor/execafterdownload.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index 33ddd16db..1c51e53da 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -1,6 +1,8 @@ from __future__ import unicode_literals import subprocess +import sys +import numbers from .common import PostProcessor from ..compat import compat_shlex_quote @@ -17,14 +19,15 @@ class ExecAfterDownloadPP(PostProcessor): def run(self, information): cmd = self.exec_cmd - if '{}' not in cmd: - cmd += ' {}' + info = {} - # expose the playlist_index and title variables to exec argument - # youtube-dl -x -o "%(playlist_index)s - %(title)s.%(ext)s" --exec "id3v2 -T {playlist_index} -t {title} {}" PLAYLIST_ID - cmd = cmd.replace('{}', compat_shlex_quote(information['filepath'])) - cmd = cmd.replace('{title}', compat_shlex_quote(information['title'])) - cmd = cmd.replace('{playlist_index}', str(information['playlist_index'])) + for key in information: + value = information[key] + info[key] = compat_shlex_quote(value) if isinstance(value, (str, unicode)) else value + + # expose info to exec argument + # youtube-dl -x -o "%(playlist_index)s - %(title)s.%(ext)s" --exec "id3v2 -T {0[playlist_index]} -t {0[title]} {0[filepath]}" PLAYLIST_ID + cmd = cmd.format(info) self._downloader.to_screen('[exec] Executing command: %s' % cmd) retCode = subprocess.call(encodeArgument(cmd), shell=True) From 3c3e4b0178cbcd3aa9f3b4c092f147c58997a73c Mon Sep 17 00:00:00 2001 From: joncody Date: Sat, 7 Apr 2018 19:02:12 -0700 Subject: [PATCH 3/9] Removed unnecessary imports --- youtube_dl/postprocessor/execafterdownload.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index 1c51e53da..da60671d2 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -1,8 +1,6 @@ from __future__ import unicode_literals import subprocess -import sys -import numbers from .common import PostProcessor from ..compat import compat_shlex_quote From 1de1510a139155188add4efd89193a7dedfd1ae8 Mon Sep 17 00:00:00 2001 From: joncody Date: Sat, 7 Apr 2018 19:13:01 -0700 Subject: [PATCH 4/9] Support python2 and python3 --- youtube_dl/postprocessor/execafterdownload.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index da60671d2..0d19f1d1e 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import subprocess +import sys from .common import PostProcessor from ..compat import compat_shlex_quote @@ -17,14 +18,14 @@ class ExecAfterDownloadPP(PostProcessor): def run(self, information): cmd = self.exec_cmd - info = {} - - for key in information: - value = information[key] - info[key] = compat_shlex_quote(value) if isinstance(value, (str, unicode)) else value # expose info to exec argument # youtube-dl -x -o "%(playlist_index)s - %(title)s.%(ext)s" --exec "id3v2 -T {0[playlist_index]} -t {0[title]} {0[filepath]}" PLAYLIST_ID + str_types = (str) if sys.version_info.major > 2 else (str, unicode) + info = {} + for key in information: + value = information[key] + info[key] = compat_shlex_quote(value) if isinstance(value, str_types) else value cmd = cmd.format(info) self._downloader.to_screen('[exec] Executing command: %s' % cmd) From 20584cede3be4b6003761e328836968d6dca4474 Mon Sep 17 00:00:00 2001 From: joncody Date: Sat, 7 Apr 2018 20:20:27 -0700 Subject: [PATCH 5/9] Using the same formatting as -o --- youtube_dl/postprocessor/execafterdownload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index 0d19f1d1e..24af1d916 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -20,13 +20,13 @@ class ExecAfterDownloadPP(PostProcessor): cmd = self.exec_cmd # expose info to exec argument - # youtube-dl -x -o "%(playlist_index)s - %(title)s.%(ext)s" --exec "id3v2 -T {0[playlist_index]} -t {0[title]} {0[filepath]}" PLAYLIST_ID + # youtube-dl -x -o "%(playlist_index)s - %(title)s.%(ext)s" --exec "id3v2 -T %(playlist_index)s -t %(title)s %(filepath)s" PLAYLIST_ID str_types = (str) if sys.version_info.major > 2 else (str, unicode) info = {} for key in information: value = information[key] info[key] = compat_shlex_quote(value) if isinstance(value, str_types) else value - cmd = cmd.format(info) + cmd = cmd % info self._downloader.to_screen('[exec] Executing command: %s' % cmd) retCode = subprocess.call(encodeArgument(cmd), shell=True) From e078f7d46892e0a40fb6cf55fb0d6072f824aee2 Mon Sep 17 00:00:00 2001 From: joncody Date: Sat, 7 Apr 2018 20:28:03 -0700 Subject: [PATCH 6/9] . --- youtube_dl/postprocessor/execafterdownload.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index 24af1d916..08292e123 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -18,14 +18,16 @@ class ExecAfterDownloadPP(PostProcessor): def run(self, information): cmd = self.exec_cmd - - # expose info to exec argument - # youtube-dl -x -o "%(playlist_index)s - %(title)s.%(ext)s" --exec "id3v2 -T %(playlist_index)s -t %(title)s %(filepath)s" PLAYLIST_ID str_types = (str) if sys.version_info.major > 2 else (str, unicode) info = {} + + if '%(filepath)s' not in cmd: + cmd += ' %(filepath)s' + for key in information: value = information[key] info[key] = compat_shlex_quote(value) if isinstance(value, str_types) else value + cmd = cmd % info self._downloader.to_screen('[exec] Executing command: %s' % cmd) From 1416f2e7bc97d7844dcfdf1ef62a5dfca5815cea Mon Sep 17 00:00:00 2001 From: joncody Date: Sat, 7 Apr 2018 22:13:35 -0700 Subject: [PATCH 7/9] Added support for old syntax --- youtube_dl/postprocessor/execafterdownload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index 08292e123..dc13cf5cc 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -21,6 +21,7 @@ class ExecAfterDownloadPP(PostProcessor): str_types = (str) if sys.version_info.major > 2 else (str, unicode) info = {} + cmd = cmd.replace('{}', '%(filepath)s') if '%(filepath)s' not in cmd: cmd += ' %(filepath)s' From 167ca6abf9c13f47994a700cce0f4ca31c47a832 Mon Sep 17 00:00:00 2001 From: joncody Date: Sun, 8 Apr 2018 19:57:10 -0700 Subject: [PATCH 8/9] Before passing info to pp.run, an extra field pp_extras is added. This dict contains some variables needed for the code within prepare_filename, for which prepare_cmd is almost an exact copy. Before returning info from pp.run we assign the value of info['pp_extras'] to a regular variable extras and call del info['pp_extras']. --- youtube_dl/YoutubeDL.py | 5 + youtube_dl/postprocessor/execafterdownload.py | 121 ++++++++++++++++-- 2 files changed, 113 insertions(+), 13 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 523dd1f7d..29f183892 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -2025,6 +2025,11 @@ class YoutubeDL(object): """Run all the postprocessors on the given file.""" info = dict(ie_info) info['filepath'] = filename + info['pp_extras'] = { + 'params': dict(self.params), + '_num_downloads': self._num_downloads, + '_NUMERIC_FIELDS': self._NUMERIC_FIELDS, + } pps_chain = [] if ie_info.get('__postprocessors') is not None: pps_chain.extend(ie_info['__postprocessors']) diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index dc13cf5cc..2c1e340d6 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -1,12 +1,25 @@ from __future__ import unicode_literals +import collections +import random +import re import subprocess import sys +import time from .common import PostProcessor -from ..compat import compat_shlex_quote +from ..compat import ( + compat_numeric_types, + compat_shlex_quote, + compat_str, +) +from string import ascii_letters from ..utils import ( encodeArgument, + encodeFilename, + expand_path, + preferredencoding, + sanitize_filename, PostProcessingError, ) @@ -16,20 +29,102 @@ class ExecAfterDownloadPP(PostProcessor): super(ExecAfterDownloadPP, self).__init__(downloader) self.exec_cmd = exec_cmd + def prepare_cmd(self, info_dict, extras): + """Generate the command.""" + try: + template_dict = dict(info_dict) + + template_dict['epoch'] = int(time.time()) + autonumber_size = extras['params'].get('autonumber_size') + if autonumber_size is None: + autonumber_size = 5 + template_dict['autonumber'] = extras['params'].get('autonumber_start', 1) - 1 + extras['_num_downloads'] + if template_dict.get('resolution') is None: + if template_dict.get('width') and template_dict.get('height'): + template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height']) + elif template_dict.get('height'): + template_dict['resolution'] = '%sp' % template_dict['height'] + elif template_dict.get('width'): + template_dict['resolution'] = '%dx?' % template_dict['width'] + + sanitize = lambda k, v: sanitize_filename( + compat_shlex_quote(compat_str(v)), + restricted=extras['params'].get('restrictfilenames'), + is_id=(k == 'id' or k.endswith('_id'))) + template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v)) + for k, v in template_dict.items() + if v is not None and not isinstance(v, (list, tuple, dict))) + template_dict = collections.defaultdict(lambda: 'NA', template_dict) + + cmdtmpl = self.exec_cmd + cmdtmpl = cmdtmpl.replace('{}', '%(filepath)s') + if '%(filepath)s' not in cmdtmpl: + cmdtmpl += ' %(filepath)s' + + # For fields playlist_index and autonumber convert all occurrences + # of %(field)s to %(field)0Nd for backward compatibility + field_size_compat_map = { + 'playlist_index': len(str(template_dict['n_entries'])), + 'autonumber': autonumber_size, + } + FIELD_SIZE_COMPAT_RE = r'(?autonumber|playlist_index)\)s' + mobj = re.search(FIELD_SIZE_COMPAT_RE, cmdtmpl) + if mobj: + cmdtmpl = re.sub( + FIELD_SIZE_COMPAT_RE, + r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')], + cmdtmpl) + + # Missing numeric fields used together with integer presentation types + # in format specification will break the argument substitution since + # string 'NA' is returned for missing fields. We will patch output + # template for missing fields to meet string presentation type. + for numeric_field in extras['_NUMERIC_FIELDS']: + if numeric_field not in template_dict: + # As of [1] format syntax is: + # %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type + # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting + FORMAT_RE = r'''(?x) + (? 2 else (str, unicode) - info = {} + extras = dict(information['pp_extras']) + del information['pp_extras'] - cmd = cmd.replace('{}', '%(filepath)s') - if '%(filepath)s' not in cmd: - cmd += ' %(filepath)s' - - for key in information: - value = information[key] - info[key] = compat_shlex_quote(value) if isinstance(value, str_types) else value - - cmd = cmd % info + cmd = self.prepare_cmd(information, extras) self._downloader.to_screen('[exec] Executing command: %s' % cmd) retCode = subprocess.call(encodeArgument(cmd), shell=True) From d642fa904b43f11f0f2d5ad4ee29ca9a5cd8059c Mon Sep 17 00:00:00 2001 From: joncody Date: Sun, 8 Apr 2018 20:51:20 -0700 Subject: [PATCH 9/9] Pass err.message to PostProcessingError --- youtube_dl/postprocessor/execafterdownload.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index 2c1e340d6..343af4ac6 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -23,7 +23,6 @@ from ..utils import ( PostProcessingError, ) - class ExecAfterDownloadPP(PostProcessor): def __init__(self, downloader, exec_cmd): super(ExecAfterDownloadPP, self).__init__(downloader) @@ -118,7 +117,7 @@ class ExecAfterDownloadPP(PostProcessor): cmd = encodeFilename(cmd, True).decode(preferredencoding()) return cmd except ValueError as err: - raise PostProcessingError(err) + raise PostProcessingError(err.message) def run(self, information): extras = dict(information['pp_extras'])