From 63000dd01ff78c1ee31d1d15dcbc8f6eb4e29540 Mon Sep 17 00:00:00 2001 From: singh-pratyush96 Date: Mon, 25 Jul 2016 01:51:52 +0530 Subject: [PATCH 1/8] Added random sleep for given interval range. Fixes #9930 --- README.md | 7 +++++-- youtube_dl/YoutubeDL.py | 6 +++++- youtube_dl/downloader/common.py | 13 +++++++++++-- youtube_dl/options.py | 2 +- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a9f3001a6..4820c9639 100644 --- a/README.md +++ b/README.md @@ -329,8 +329,11 @@ which means you can modify it, redistribute it or use it however you like. --bidi-workaround Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH - --sleep-interval SECONDS Number of seconds to sleep before each - download. + --sleep-interval SECONDS/RANGE Number of seconds to sleep before each + download. Also accepts random sleep in + format to and + to, where limits + are in seconds. ## Video Format Options: -f, --format FORMAT Video format code, see the "FORMAT diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 6551f086f..617b00cf3 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -250,6 +250,10 @@ class YoutubeDL(object): call_home: Boolean, true iff we are allowed to contact the youtube-dl servers for debugging. sleep_interval: Number of seconds to sleep before each download. + Accepts range for random sleep in formats: + - to + - to + where limits are in seconds listformats: Print an overview of available video formats and exit. list_thumbnails: Print a table of all thumbnails and exit. match_filter: A function that gets called with the info_dict of @@ -1684,7 +1688,7 @@ class YoutubeDL(object): except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_error('unable to download video data: %s' % error_to_compat_str(err)) return - except (OSError, IOError) as err: + except (OSError, IOError, ValueError) as err: raise UnavailableVideoError(err) except (ContentTooShortError, ) as err: self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 1dba9f49a..5c3a5ca06 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -4,6 +4,7 @@ import os import re import sys import time +import random from ..compat import compat_os_name from ..utils import ( @@ -344,8 +345,16 @@ class FileDownloader(object): sleep_interval = self.params.get('sleep_interval') if sleep_interval: - self.to_screen('[download] Sleeping %s seconds...' % sleep_interval) - time.sleep(sleep_interval) + sleep_interval_lower = sleep_interval.lower() + + if 'to' in sleep_interval_lower: + lower_sleep_limit, upper_sleep_limit = map(float, sleep_interval_lower.split('to')) + sleep_time = random.uniform(lower_sleep_limit, upper_sleep_limit) + else: + sleep_time = float(sleep_interval) + + self.to_screen('[download] Sleeping %s seconds...' % sleep_time) + time.sleep(sleep_time) return self.real_download(filename, info_dict) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index c4a85b2c0..18a68867d 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -489,7 +489,7 @@ def parseOpts(overrideArguments=None): help='Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH') workarounds.add_option( '--sleep-interval', metavar='SECONDS', - dest='sleep_interval', type=float, + dest='sleep_interval', type=str, help='Number of seconds to sleep before each download.') verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') From 6ab2759a7cd797e8032ea328dea6737899375bb4 Mon Sep 17 00:00:00 2001 From: singh-pratyush96 Date: Mon, 25 Jul 2016 09:41:21 +0530 Subject: [PATCH 2/8] Added option --max-sleep-interval which activates random sleep if set. --- youtube_dl/YoutubeDL.py | 8 ++++---- youtube_dl/__init__.py | 1 + youtube_dl/downloader/common.py | 23 +++++++++++++++-------- youtube_dl/options.py | 10 ++++++++-- youtube_dl/utils.py | 9 +++++++++ 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 617b00cf3..7e65f305a 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -76,6 +76,7 @@ from .utils import ( std_headers, subtitles_filename, UnavailableVideoError, + InvalidSleepTimeError, url_basename, version_tuple, write_json_file, @@ -250,10 +251,7 @@ class YoutubeDL(object): call_home: Boolean, true iff we are allowed to contact the youtube-dl servers for debugging. sleep_interval: Number of seconds to sleep before each download. - Accepts range for random sleep in formats: - - to - - to - where limits are in seconds + max_sleep_interval Upper bound of number of seconds for random sleep. listformats: Print an overview of available video formats and exit. list_thumbnails: Print a table of all thumbnails and exit. match_filter: A function that gets called with the info_dict of @@ -1782,6 +1780,8 @@ class YoutubeDL(object): except MaxDownloadsReached: self.to_screen('[info] Maximum number of downloaded files reached.') raise + except InvalidSleepTimeError: + self.report_error('Invalid argument for sleep') else: if self.params.get('dump_single_json', False): self.to_stdout(json.dumps(res)) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 2b34bf9c2..309f156da 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -370,6 +370,7 @@ def _real_main(argv=None): 'source_address': opts.source_address, 'call_home': opts.call_home, 'sleep_interval': opts.sleep_interval, + 'max_sleep_interval': opts.max_sleep_interval, 'external_downloader': opts.external_downloader, 'list_thumbnails': opts.list_thumbnails, 'playlist_items': opts.playlist_items, diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 5c3a5ca06..8747186c5 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -13,6 +13,7 @@ from ..utils import ( decodeArgument, format_bytes, timeconvert, + InvalidSleepTimeError ) @@ -345,16 +346,22 @@ class FileDownloader(object): sleep_interval = self.params.get('sleep_interval') if sleep_interval: - sleep_interval_lower = sleep_interval.lower() + min_sleep_interval = sleep_interval + max_sleep_interval = self.params.get('max_sleep_interval') - if 'to' in sleep_interval_lower: - lower_sleep_limit, upper_sleep_limit = map(float, sleep_interval_lower.split('to')) - sleep_time = random.uniform(lower_sleep_limit, upper_sleep_limit) - else: - sleep_time = float(sleep_interval) + try: + if max_sleep_interval: + sleep_time = random.uniform( + float(min_sleep_interval), + float(max_sleep_interval) + ) + else: + sleep_time = float(min_sleep_interval) - self.to_screen('[download] Sleeping %s seconds...' % sleep_time) - time.sleep(sleep_time) + self.to_screen('[download] Sleeping %s seconds...' % sleep_time) + time.sleep(sleep_time) + except (ValueError, IOError) as err: + raise InvalidSleepTimeError() return self.real_download(filename, info_dict) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 18a68867d..290e973d7 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -489,8 +489,14 @@ def parseOpts(overrideArguments=None): help='Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH') workarounds.add_option( '--sleep-interval', metavar='SECONDS', - dest='sleep_interval', type=str, - help='Number of seconds to sleep before each download.') + dest='sleep_interval', type=float, + help='Number of seconds to sleep before each download.', + ) + workarounds.add_option( + '--max-sleep-interval', metavar='SECONDS', + dest='max_sleep_interval', type=float, + help='Maximum sleep time for random sleep.' + ) verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') verbosity.add_option( diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index f5cd6819b..ad46277a4 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -752,6 +752,15 @@ class UnavailableVideoError(Exception): pass +class InvalidSleepTimeError(Exception): + """Invalid sleep time provided exception + + This exception will be thrown when user provides invalid + time for sleep in --sleep-interval or --max-sleep-interval. + """ + pass + + class ContentTooShortError(Exception): """Content Too Short exception. From 5c6dfd34150296f5569a82c4b1fdc35180d176b0 Mon Sep 17 00:00:00 2001 From: singh-pratyush96 Date: Mon, 25 Jul 2016 09:47:39 +0530 Subject: [PATCH 3/8] Reverted changes in README --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4820c9639..0deaa7673 100644 --- a/README.md +++ b/README.md @@ -329,12 +329,8 @@ which means you can modify it, redistribute it or use it however you like. --bidi-workaround Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH - --sleep-interval SECONDS/RANGE Number of seconds to sleep before each - download. Also accepts random sleep in - format to and - to, where limits - are in seconds. - + --sleep-interval SECONDS Number of seconds to sleep before each + download. ## Video Format Options: -f, --format FORMAT Video format code, see the "FORMAT SELECTION" for all the info From ada2d3f3b2b79715d62dfd749fab5c2057352475 Mon Sep 17 00:00:00 2001 From: singh-pratyush96 Date: Mon, 25 Jul 2016 09:52:36 +0530 Subject: [PATCH 4/8] Added missing colon to max_sleep_interval. --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 7e65f305a..015d29708 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -251,7 +251,7 @@ class YoutubeDL(object): call_home: Boolean, true iff we are allowed to contact the youtube-dl servers for debugging. sleep_interval: Number of seconds to sleep before each download. - max_sleep_interval Upper bound of number of seconds for random sleep. + max_sleep_interval:Upper bound of number of seconds for random sleep. listformats: Print an overview of available video formats and exit. list_thumbnails: Print a table of all thumbnails and exit. match_filter: A function that gets called with the info_dict of From 8b750c835420c2820098a9c11e3baf38434b71c2 Mon Sep 17 00:00:00 2001 From: singh-pratyush96 Date: Mon, 25 Jul 2016 11:25:17 +0530 Subject: [PATCH 5/8] Removed runtime checks for sleep intervals and corresponding Exception subclass. --- youtube_dl/YoutubeDL.py | 3 --- youtube_dl/downloader/common.py | 21 +++++++-------------- youtube_dl/utils.py | 9 --------- 3 files changed, 7 insertions(+), 26 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 015d29708..685c20e67 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -76,7 +76,6 @@ from .utils import ( std_headers, subtitles_filename, UnavailableVideoError, - InvalidSleepTimeError, url_basename, version_tuple, write_json_file, @@ -1780,8 +1779,6 @@ class YoutubeDL(object): except MaxDownloadsReached: self.to_screen('[info] Maximum number of downloaded files reached.') raise - except InvalidSleepTimeError: - self.report_error('Invalid argument for sleep') else: if self.params.get('dump_single_json', False): self.to_stdout(json.dumps(res)) diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 8747186c5..075c425be 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -13,7 +13,6 @@ from ..utils import ( decodeArgument, format_bytes, timeconvert, - InvalidSleepTimeError ) @@ -347,21 +346,15 @@ class FileDownloader(object): sleep_interval = self.params.get('sleep_interval') if sleep_interval: min_sleep_interval = sleep_interval - max_sleep_interval = self.params.get('max_sleep_interval') + max_sleep_interval = self.params.get('max_sleep_interval') or min_sleep_interval - try: - if max_sleep_interval: - sleep_time = random.uniform( - float(min_sleep_interval), - float(max_sleep_interval) - ) - else: - sleep_time = float(min_sleep_interval) + sleep_time = random.uniform( + min_sleep_interval, + max_sleep_interval + ) - self.to_screen('[download] Sleeping %s seconds...' % sleep_time) - time.sleep(sleep_time) - except (ValueError, IOError) as err: - raise InvalidSleepTimeError() + self.to_screen('[download] Sleeping %s seconds...' % sleep_time) + time.sleep(sleep_time) return self.real_download(filename, info_dict) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index ad46277a4..f5cd6819b 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -752,15 +752,6 @@ class UnavailableVideoError(Exception): pass -class InvalidSleepTimeError(Exception): - """Invalid sleep time provided exception - - This exception will be thrown when user provides invalid - time for sleep in --sleep-interval or --max-sleep-interval. - """ - pass - - class ContentTooShortError(Exception): """Content Too Short exception. From 9610b62fb1e32e67635704324113ebb8e5497755 Mon Sep 17 00:00:00 2001 From: singh-pratyush96 Date: Mon, 25 Jul 2016 12:13:08 +0530 Subject: [PATCH 6/8] Removed ValueError catch --- README.md | 1 + youtube_dl/YoutubeDL.py | 2 +- youtube_dl/options.py | 3 +-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0deaa7673..a9f3001a6 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,7 @@ which means you can modify it, redistribute it or use it however you like. or fribidi executable in PATH --sleep-interval SECONDS Number of seconds to sleep before each download. + ## Video Format Options: -f, --format FORMAT Video format code, see the "FORMAT SELECTION" for all the info diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 685c20e67..10dc86038 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1685,7 +1685,7 @@ class YoutubeDL(object): except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_error('unable to download video data: %s' % error_to_compat_str(err)) return - except (OSError, IOError, ValueError) as err: + except (OSError, IOError) as err: raise UnavailableVideoError(err) except (ContentTooShortError, ) as err: self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 290e973d7..8c6207219 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -490,8 +490,7 @@ def parseOpts(overrideArguments=None): workarounds.add_option( '--sleep-interval', metavar='SECONDS', dest='sleep_interval', type=float, - help='Number of seconds to sleep before each download.', - ) + help='Number of seconds to sleep before each download.') workarounds.add_option( '--max-sleep-interval', metavar='SECONDS', dest='max_sleep_interval', type=float, From b6e542a3df19d973ee6a406c67fac1dd8b5b5947 Mon Sep 17 00:00:00 2001 From: singh-pratyush96 Date: Mon, 25 Jul 2016 01:51:52 +0530 Subject: [PATCH 7/8] Added random sleep for given interval range. Fixes #9930 --- youtube_dl/YoutubeDL.py | 1 + youtube_dl/__init__.py | 1 + youtube_dl/downloader/common.py | 13 +++++++++++-- youtube_dl/options.py | 5 +++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 6551f086f..10dc86038 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -250,6 +250,7 @@ class YoutubeDL(object): call_home: Boolean, true iff we are allowed to contact the youtube-dl servers for debugging. sleep_interval: Number of seconds to sleep before each download. + max_sleep_interval:Upper bound of number of seconds for random sleep. listformats: Print an overview of available video formats and exit. list_thumbnails: Print a table of all thumbnails and exit. match_filter: A function that gets called with the info_dict of diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 2b34bf9c2..309f156da 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -370,6 +370,7 @@ def _real_main(argv=None): 'source_address': opts.source_address, 'call_home': opts.call_home, 'sleep_interval': opts.sleep_interval, + 'max_sleep_interval': opts.max_sleep_interval, 'external_downloader': opts.external_downloader, 'list_thumbnails': opts.list_thumbnails, 'playlist_items': opts.playlist_items, diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 1dba9f49a..075c425be 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -4,6 +4,7 @@ import os import re import sys import time +import random from ..compat import compat_os_name from ..utils import ( @@ -344,8 +345,16 @@ class FileDownloader(object): sleep_interval = self.params.get('sleep_interval') if sleep_interval: - self.to_screen('[download] Sleeping %s seconds...' % sleep_interval) - time.sleep(sleep_interval) + min_sleep_interval = sleep_interval + max_sleep_interval = self.params.get('max_sleep_interval') or min_sleep_interval + + sleep_time = random.uniform( + min_sleep_interval, + max_sleep_interval + ) + + self.to_screen('[download] Sleeping %s seconds...' % sleep_time) + time.sleep(sleep_time) return self.real_download(filename, info_dict) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index c4a85b2c0..8c6207219 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -491,6 +491,11 @@ def parseOpts(overrideArguments=None): '--sleep-interval', metavar='SECONDS', dest='sleep_interval', type=float, help='Number of seconds to sleep before each download.') + workarounds.add_option( + '--max-sleep-interval', metavar='SECONDS', + dest='max_sleep_interval', type=float, + help='Maximum sleep time for random sleep.' + ) verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') verbosity.add_option( From 0fcbebeef0df8572412ff025d5f46011585c8d41 Mon Sep 17 00:00:00 2001 From: Pratyush Singh Date: Mon, 25 Jul 2016 16:59:28 +0530 Subject: [PATCH 8/8] Fixed wrongly declared max_sleep_interal --- youtube_dl/downloader/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 075c425be..7714f111f 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -346,7 +346,7 @@ class FileDownloader(object): sleep_interval = self.params.get('sleep_interval') if sleep_interval: min_sleep_interval = sleep_interval - max_sleep_interval = self.params.get('max_sleep_interval') or min_sleep_interval + max_sleep_interval = self.params.get('max_sleep_interval') if self.params.get('max_sleep_interval') else min_sleep_interval sleep_time = random.uniform( min_sleep_interval,