From 28bc56459a0e8f3dc5e016b3a45a31f9f0ad0906 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 9 Jun 2016 22:23:11 +0800 Subject: [PATCH 1/4] [utils] Add DownloadTarget --- youtube_dl/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 229de4b39..f7e6ad970 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2834,3 +2834,9 @@ def decode_packed_codes(code): return re.sub( r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)], obfucasted_code) + + +class DownloadTarget(object): + MEDIA = 1 + THUMBNAILS = 2 + SUBTITLES = 3 From 3799177bdb0ebe62bb41bfaba83c1fdb408658be Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 9 Jun 2016 22:28:10 +0800 Subject: [PATCH 2/4] [postprocessor] Add DEPENDENCY markers --- youtube_dl/postprocessor/embedthumbnail.py | 3 +++ youtube_dl/postprocessor/execafterdownload.py | 7 ++++++- youtube_dl/postprocessor/ffmpeg.py | 19 +++++++++++++++++++ youtube_dl/postprocessor/metadatafromtitle.py | 2 ++ youtube_dl/postprocessor/xattrpp.py | 2 ++ 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py index 3bad5a266..1e1e4ff8b 100644 --- a/youtube_dl/postprocessor/embedthumbnail.py +++ b/youtube_dl/postprocessor/embedthumbnail.py @@ -9,6 +9,7 @@ from .ffmpeg import FFmpegPostProcessor from ..utils import ( check_executable, + DownloadTarget, encodeArgument, encodeFilename, PostProcessingError, @@ -22,6 +23,8 @@ class EmbedThumbnailPPError(PostProcessingError): class EmbedThumbnailPP(FFmpegPostProcessor): + DEPENDENCY = set([DownloadTarget.MEDIA, DownloadTarget.THUMBNAILS]) + def __init__(self, downloader=None, already_have_thumbnail=False): super(EmbedThumbnailPP, self).__init__(downloader) self._already_have_thumbnail = already_have_thumbnail diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index 90630c2d7..89e2f6de8 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -4,10 +4,15 @@ import subprocess from .common import PostProcessor from ..compat import compat_shlex_quote -from ..utils import PostProcessingError +from ..utils import ( + DownloadTarget, + PostProcessingError, +) class ExecAfterDownloadPP(PostProcessor): + DEPENDENCY = set([DownloadTarget.MEDIA]) + def __init__(self, downloader, exec_cmd): super(ExecAfterDownloadPP, self).__init__(downloader) self.exec_cmd = exec_cmd diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index fa99b0c2a..3efd3d88b 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -12,6 +12,7 @@ from ..compat import ( compat_subprocess_get_DEVNULL, ) from ..utils import ( + DownloadTarget, encodeArgument, encodeFilename, get_exe_version, @@ -180,6 +181,8 @@ class FFmpegPostProcessor(PostProcessor): class FFmpegExtractAudioPP(FFmpegPostProcessor): + DEPENDENCY = set([DownloadTarget.MEDIA]) + def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False): FFmpegPostProcessor.__init__(self, downloader) if preferredcodec is None: @@ -308,6 +311,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): class FFmpegVideoConvertorPP(FFmpegPostProcessor): + DEPENDENCY = set([DownloadTarget.MEDIA]) + def __init__(self, downloader=None, preferedformat=None): super(FFmpegVideoConvertorPP, self).__init__(downloader) self._preferedformat = preferedformat @@ -331,6 +336,8 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor): class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): + DEPENDENCY = set([DownloadTarget.MEDIA, DownloadTarget.SUBTITLES]) + def run(self, information): if information['ext'] not in ('mp4', 'webm', 'mkv'): self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4, webm or mkv files') @@ -387,6 +394,8 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): class FFmpegMetadataPP(FFmpegPostProcessor): + DEPENDENCY = set([DownloadTarget.MEDIA]) + def run(self, info): metadata = {} @@ -437,6 +446,8 @@ class FFmpegMetadataPP(FFmpegPostProcessor): class FFmpegMergerPP(FFmpegPostProcessor): + DEPENDENCY = set([DownloadTarget.MEDIA]) + def run(self, info): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') @@ -465,6 +476,8 @@ class FFmpegMergerPP(FFmpegPostProcessor): class FFmpegFixupStretchedPP(FFmpegPostProcessor): + DEPENDENCY = set([DownloadTarget.MEDIA]) + def run(self, info): stretched_ratio = info.get('stretched_ratio') if stretched_ratio is None or stretched_ratio == 1: @@ -484,6 +497,8 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor): class FFmpegFixupM4aPP(FFmpegPostProcessor): + DEPENDENCY = set([DownloadTarget.MEDIA]) + def run(self, info): if info.get('container') != 'm4a_dash': return [], info @@ -502,6 +517,8 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor): class FFmpegFixupM3u8PP(FFmpegPostProcessor): + DEPENDENCY = set([DownloadTarget.MEDIA]) + def run(self, info): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') @@ -517,6 +534,8 @@ class FFmpegFixupM3u8PP(FFmpegPostProcessor): class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): + DEPENDENCY = set([DownloadTarget.SUBTITLES]) + def __init__(self, downloader=None, format=None): super(FFmpegSubtitlesConvertorPP, self).__init__(downloader) self.format = format diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py index 42377fa0f..9078fcaf1 100644 --- a/youtube_dl/postprocessor/metadatafromtitle.py +++ b/youtube_dl/postprocessor/metadatafromtitle.py @@ -11,6 +11,8 @@ class MetadataFromTitlePPError(PostProcessingError): class MetadataFromTitlePP(PostProcessor): + DEPENDENCY = set() + def __init__(self, downloader, titleformat): super(MetadataFromTitlePP, self).__init__(downloader) self._titleformat = titleformat diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py index e39ca60aa..ae3db7edf 100644 --- a/youtube_dl/postprocessor/xattrpp.py +++ b/youtube_dl/postprocessor/xattrpp.py @@ -9,6 +9,7 @@ from .common import PostProcessor from ..compat import compat_os_name from ..utils import ( check_executable, + DownloadTarget, hyphenate_date, version_tuple, PostProcessingError, @@ -33,6 +34,7 @@ class XAttrMetadataError(PostProcessingError): class XAttrMetadataPP(PostProcessor): + DEPENDENCY = set([DownloadTarget.MEDIA]) # # More info about extended attributes for media: From 7c640d273c9d959457c876e635410a447bb89492 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 9 Jun 2016 22:41:11 +0800 Subject: [PATCH 3/4] [YoutubeDL] Check download targets --- youtube_dl/YoutubeDL.py | 42 ++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 5036289b0..de4430110 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -48,6 +48,7 @@ from .utils import ( determine_ext, determine_protocol, DownloadError, + DownloadTarget, encode_compat_str, encodeFilename, error_to_compat_str, @@ -1462,6 +1463,7 @@ class YoutubeDL(object): self._num_downloads += 1 info_dict['_filename'] = filename = self.prepare_filename(info_dict) + info_dict['_downloaded_targets'] = [] # Forced printings if self.params.get('forcetitle', False): @@ -1541,6 +1543,9 @@ class YoutubeDL(object): # that way it will silently go on when used with unsupporting IE subtitles = info_dict['requested_subtitles'] ie = self.get_info_extractor(info_dict['extractor_key']) + + all_subtitles_ok = True + for sub_lang, sub_info in subtitles.items(): sub_format = sub_info['ext'] if sub_info.get('data') is not None: @@ -1552,6 +1557,7 @@ class YoutubeDL(object): except ExtractorError as err: self.report_warning('Unable to download subtitle for "%s": %s' % (sub_lang, error_to_compat_str(err.cause))) + all_subtitles_ok = False continue try: sub_filename = subtitles_filename(filename, sub_lang, sub_format) @@ -1563,8 +1569,12 @@ class YoutubeDL(object): subfile.write(sub_data) except (OSError, IOError): self.report_error('Cannot write subtitles file ' + sub_filename) + all_subtitles_ok = False return + if all_subtitles_ok: + info_dict['_downloaded_targets'].append(DownloadTarget.SUBTITLES) + if self.params.get('writeinfojson', False): infofn = replace_extension(filename, 'info.json', info_dict.get('ext')) if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): @@ -1577,7 +1587,8 @@ class YoutubeDL(object): self.report_error('Cannot write metadata to JSON file ' + infofn) return - self._write_thumbnails(info_dict, filename) + if self._write_thumbnails(info_dict, filename): + info_dict['_downloaded_targets'].append(DownloadTarget.THUMBNAILS) if not self.params.get('skip_download', False): try: @@ -1718,12 +1729,16 @@ class YoutubeDL(object): else: assert fixup_policy in ('ignore', 'never') - try: - self.post_process(filename, info_dict) - except (PostProcessingError) as err: - self.report_error('postprocessing: %s' % str(err)) - return - self.record_download_archive(info_dict) + info_dict['_downloaded_targets'].append(DownloadTarget.MEDIA) + + try: + self.post_process(filename, info_dict) + except (PostProcessingError) as err: + self.report_error('postprocessing: %s' % str(err)) + return + + if DownloadTarget.MEDIA in info_dict['_downloaded_targets']: + self.record_download_archive(info_dict) def download(self, url_list): """Download a given list of URLs.""" @@ -1776,11 +1791,15 @@ class YoutubeDL(object): """Run all the postprocessors on the given file.""" info = dict(ie_info) info['filepath'] = filename + downloaded_targets = set(ie_info['_downloaded_targets']) + pps_chain = [] if ie_info.get('__postprocessors') is not None: pps_chain.extend(ie_info['__postprocessors']) pps_chain.extend(self._pps) for pp in pps_chain: + if not pp.DEPENDENCY.issubset(downloaded_targets): + continue files_to_delete = [] try: files_to_delete, info = pp.run(info) @@ -2093,11 +2112,13 @@ class YoutubeDL(object): elif self.params.get('write_all_thumbnails', False): thumbnails = info_dict.get('thumbnails') else: - return + return False if not thumbnails: # No thumbnails present, so return immediately - return + return False + + all_thumbnails_ok = True for t in thumbnails: thumb_ext = determine_ext(t['url'], 'jpg') @@ -2120,3 +2141,6 @@ class YoutubeDL(object): except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_warning('Unable to download thumbnail "%s": %s' % (t['url'], error_to_compat_str(err))) + all_thumbnails_ok = False + + return all_thumbnails_ok From 7ec90b9898b88d7160c6159e6f20da8bc7164138 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 9 Jun 2016 22:47:10 +0800 Subject: [PATCH 4/4] [__init__] Don't download thumbnails if --skip-download is used --- youtube_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 4905674ad..4a0c9c179 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -252,7 +252,7 @@ def _real_main(argv=None): 'key': 'EmbedThumbnail', 'already_have_thumbnail': already_have_thumbnail }) - if not already_have_thumbnail: + if not already_have_thumbnail and not opts.skip_download: opts.writethumbnail = True # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way. # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.