From 2915ae8eb66a10ce6064588f39779f0dfa7abe6a Mon Sep 17 00:00:00 2001 From: Mickey Rose Date: Tue, 23 Jun 2020 12:07:45 +0200 Subject: [PATCH 1/2] [HttpFD] extract method create_context --- youtube_dl/downloader/http.py | 42 +++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index 5046878df..f2cd0cb76 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -24,31 +24,19 @@ from ..utils import ( ) +class DownloadContext(dict): + __getattr__ = dict.get + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__ + + class HttpFD(FileDownloader): - def real_download(self, filename, info_dict): - url = info_dict['url'] - - class DownloadContext(dict): - __getattr__ = dict.get - __setattr__ = dict.__setitem__ - __delattr__ = dict.__delitem__ - + def create_context(self, filename, info_dict): ctx = DownloadContext() ctx.filename = filename ctx.tmpfilename = self.temp_name(filename) ctx.stream = None - # Do not include the Accept-Encoding header - headers = {'Youtubedl-no-compression': 'True'} - add_headers = info_dict.get('http_headers') - if add_headers: - headers.update(add_headers) - - is_test = self.params.get('test', False) - chunk_size = self._TEST_FILE_SIZE if is_test else ( - info_dict.get('downloader_options', {}).get('http_chunk_size') - or self.params.get('http_chunk_size') or 0) - ctx.open_mode = 'wb' ctx.resume_len = 0 ctx.data_len = None @@ -63,6 +51,22 @@ class HttpFD(FileDownloader): encodeFilename(ctx.tmpfilename)) ctx.is_resume = ctx.resume_len > 0 + return ctx + + def real_download(self, filename, info_dict): + url = info_dict['url'] + ctx = self.create_context(filename, info_dict) + + # Do not include the Accept-Encoding header + headers = {'Youtubedl-no-compression': 'True'} + add_headers = info_dict.get('http_headers') + if add_headers: + headers.update(add_headers) + + is_test = self.params.get('test', False) + chunk_size = self._TEST_FILE_SIZE if is_test else ( + info_dict.get('downloader_options', {}).get('http_chunk_size') + or self.params.get('http_chunk_size') or 0) count = 0 retries = self.params.get('retries', 0) From 6ec9586e02f9ef4538ad8b13a4452f7ff3f1bb35 Mon Sep 17 00:00:00 2001 From: Mickey Rose Date: Tue, 23 Jun 2020 12:20:12 +0200 Subject: [PATCH 2/2] [FragmentFD] only write fragment to disk if --keep-fragments is given --- README.md | 4 ++-- youtube_dl/downloader/fragment.py | 28 +++++++++++++++++----------- youtube_dl/downloader/http.py | 25 +++++++++++++++++-------- youtube_dl/options.py | 2 +- 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 45326c69e..ce8a9a955 100644 --- a/README.md +++ b/README.md @@ -192,8 +192,8 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo --abort-on-unavailable-fragment Abort downloading when some fragment is not available --keep-fragments Keep downloaded fragments on disk after - downloading is finished; fragments are - erased by default + downloading is finished; fragments are not + saved by default --buffer-size SIZE Size of download buffer (e.g. 1024 or 16K) (default is 1024) --no-resize-buffer Do not automatically adjust the buffer diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py index 02f35459e..5f5b5f976 100644 --- a/youtube_dl/downloader/fragment.py +++ b/youtube_dl/downloader/fragment.py @@ -1,5 +1,6 @@ from __future__ import division, unicode_literals +import io import os import time import json @@ -96,18 +97,26 @@ class FragmentFD(FileDownloader): frag_index_stream.close() def _download_fragment(self, ctx, frag_url, info_dict, headers=None): - fragment_filename = '%s-Frag%d' % (ctx['tmpfilename'], ctx['fragment_index']) - success = ctx['dl'].download(fragment_filename, { + if self.params.get('keep_fragments', False): + frag_file = None + frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], ctx['fragment_index']) + else: + frag_file = io.BytesIO() + frag_filename = None + success = ctx['dl'].download(frag_file or frag_filename, { 'url': frag_url, 'http_headers': headers or info_dict.get('http_headers'), }) if not success: - return False, None - down, frag_sanitized = sanitize_open(fragment_filename, 'rb') - ctx['fragment_filename_sanitized'] = frag_sanitized - frag_content = down.read() - down.close() - return True, frag_content + frag_content = None + elif self.params.get('keep_fragments', False): + frag_file, frag_sanitized = sanitize_open(frag_filename, 'rb') + frag_content = frag_file.read() + else: + frag_content = frag_file.getvalue() + if frag_file: + frag_file.close() + return success, frag_content def _append_fragment(self, ctx, frag_content): try: @@ -116,9 +125,6 @@ class FragmentFD(FileDownloader): finally: if self.__do_ytdl_file(ctx): self._write_ytdl_file(ctx) - if not self.params.get('keep_fragments', False): - os.remove(encodeFilename(ctx['fragment_filename_sanitized'])) - del ctx['fragment_filename_sanitized'] def _prepare_frag_download(self, ctx): if 'live' not in ctx: diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index f2cd0cb76..66ead8adc 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import errno +import io import os import socket import time @@ -33,9 +34,15 @@ class DownloadContext(dict): class HttpFD(FileDownloader): def create_context(self, filename, info_dict): ctx = DownloadContext() - ctx.filename = filename - ctx.tmpfilename = self.temp_name(filename) - ctx.stream = None + + if isinstance(filename, io.IOBase): + ctx.filename = '-' + ctx.tmpfilename = '-' + ctx.stream = filename + else: + ctx.filename = filename + ctx.tmpfilename = self.temp_name(filename) + ctx.stream = None ctx.open_mode = 'wb' ctx.resume_len = 0 @@ -44,7 +51,8 @@ class HttpFD(FileDownloader): ctx.start_time = time.time() ctx.chunk_size = None - if self.params.get('continuedl', True): + if (self.params.get('continuedl', True) + and ctx.tmpfilename != '-'): # Establish possible resume length if os.path.isfile(encodeFilename(ctx.tmpfilename)): ctx.resume_len = os.path.getsize( @@ -221,11 +229,12 @@ class HttpFD(FileDownloader): before = start # start measuring def retry(e): - to_stdout = ctx.tmpfilename == '-' - if not to_stdout: + if ctx.tmpfilename == '-': + ctx.resume_len = byte_counter + else: ctx.stream.close() - ctx.stream = None - ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename)) + ctx.stream = None + ctx.resume_len = os.path.getsize(encodeFilename(ctx.tmpfilename)) raise RetryDownload(e) while True: diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 6d5ac62b3..61650f6a2 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -473,7 +473,7 @@ def parseOpts(overrideArguments=None): downloader.add_option( '--keep-fragments', action='store_true', dest='keep_fragments', default=False, - help='Keep downloaded fragments on disk after downloading is finished; fragments are erased by default') + help='Keep downloaded fragments on disk after downloading is finished; fragments are not saved by default') downloader.add_option( '--buffer-size', dest='buffersize', metavar='SIZE', default='1024',