From 989c8a28303fb5c6b55b84685fd10629c11d7664 Mon Sep 17 00:00:00 2001 From: MrS0m30n3 Date: Sun, 20 Apr 2014 19:06:15 +0300 Subject: [PATCH 1/5] Add stop/pause/resume functionality --- youtube_dl/YoutubeDL.py | 30 ++++++++++++++++++++++++++++++ youtube_dl/downloader/common.py | 27 ++++++++++++++++++++++++++- youtube_dl/downloader/http.py | 6 ++++++ youtube_dl/utils.py | 3 +++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index d4dd05d8c..a88df9cd0 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -55,6 +55,7 @@ from .utils import ( write_string, YoutubeDLHandler, prepend_extension, + StopDownloads ) from .extractor import get_info_extractor, gen_extractors from .downloader import get_suitable_downloader @@ -186,6 +187,8 @@ class YoutubeDL(object): self._ies = [] self._ies_instances = {} self._pps = [] + self._stop = False + self._pause = False self._progress_hooks = [] self._download_retcode = 0 self._num_downloads = 0 @@ -980,6 +983,9 @@ class YoutubeDL(object): fd = get_suitable_downloader(info)(self, self.params) for ph in self._progress_hooks: fd.add_progress_hook(ph) + # Add stop, pause handlers + fd.add_stop_handler(self._stop_handler) + fd.add_pause_handler(self._pause_handler) return fd.download(name, info) if info_dict.get('requested_formats') is not None: downloaded = [] @@ -1013,6 +1019,8 @@ class YoutubeDL(object): except (ContentTooShortError, ) as err: self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) return + except StopDownloads: + return if success: try: @@ -1023,6 +1031,28 @@ class YoutubeDL(object): self.record_download_archive(info_dict) + def _stop_handler(self): + """ Return self._stop status """ + return self._stop + + def _pause_handler(self): + """ Return self._pause status """ + return self._pause + + def stop(self): + """ Stop downloads """ + if self._stop: + self._stop = False + else: + self._stop = True + + def pause(self): + """ Pause/Resume downloads """ + if self._pause: + self._pause = False + else: + self._pause = True + def download(self, url_list): """Download a given list of URLs.""" if (len(url_list) > 1 and diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 917f3450e..b8ce20af0 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -8,6 +8,7 @@ from ..utils import ( encodeFilename, format_bytes, timeconvert, + StopDownloads ) @@ -49,6 +50,8 @@ class FileDownloader(object): self.ydl = ydl self._progress_hooks = [] self.params = params + self._stop_handler = None + self._pause_handler = None @staticmethod def format_seconds(seconds): @@ -155,7 +158,7 @@ class FileDownloader(object): speed = float(byte_counter) / elapsed if speed > rate_limit: time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) - + def temp_name(self, filename): """Returns a temporary filename for the given filename.""" if self.params.get('nopart', False) or filename == u'-' or \ @@ -298,6 +301,28 @@ class FileDownloader(object): for ph in self._progress_hooks: ph(status) + def _stop(self): + """ Check stop handler """ + if self._stop_handler is not None: + if self._stop_handler(): + raise StopDownloads() + + def _pause(self): + """ Check pause/resume handler """ + if self._pause_handler is not None: + while self._pause_handler(): + # Break if stop handler enable + if self._stop_handler(): break + time.sleep(1) + + def add_stop_handler(self, shand): + """ shand gets checked. If True raise StopDownloads """ + self._stop_handler = shand + + def add_pause_handler(self, phand): + """ phand gets checked. If True pause downloads """ + self._pause_handler = phand + def add_progress_hook(self, ph): """ ph gets called on download progress, with a dictionary with the entries * filename: The final filename diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index cc8b9c9a7..4f18c9e26 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -162,6 +162,12 @@ class HttpFD(FileDownloader): 'speed': speed, }) + # Check stop handler + self._stop() + + # Check pause handler + self._pause() + # Apply rate limit self.slow_down(start, byte_counter - resume_len) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 9c9320934..3daf42d99 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -655,6 +655,9 @@ class MaxDownloadsReached(Exception): """ --max-downloads limit has been reached. """ pass +class StopDownloads(Exception): + """ Stop Downloads """ + pass class UnavailableVideoError(Exception): """Unavailable Format exception. From d7235497ca8fb1d109b8af58779b5a43930910dd Mon Sep 17 00:00:00 2001 From: MrS0m30n3 Date: Sun, 20 Apr 2014 19:10:44 +0300 Subject: [PATCH 2/5] Stop/Pause/Resume example --- test_handlers.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100755 test_handlers.py diff --git a/test_handlers.py b/test_handlers.py new file mode 100755 index 000000000..f2341c0b1 --- /dev/null +++ b/test_handlers.py @@ -0,0 +1,63 @@ +#! /usr/bin/env python + +# Test stop, pause & resume handlers + +from time import sleep +from threading import Thread + +from youtube_dl import YoutubeDL + +class Downloader(Thread): + + def __init__(self, options, url): + super(Downloader, self).__init__() + self.ydl_item = YoutubeDL(options) + self.url = url + + def run(self): + self.ydl_item.add_default_info_extractors() + self.ydl_item.download([self.url]) + + @staticmethod + def format_percent(db, tb): + if db is None: return None + pr = float(db) / float(tb) * 100.0 + return '%6s' % ('%3.1f%%' % pr) + + def progress_hook(self, pr): + print Downloader.format_percent(pr.get('downloaded_bytes'), pr.get('total_bytes')) + + def download(self): + """ Call self.start() """ + self.start() + + def close(self): + """ Stop download and join thread """ + self.ydl_item.stop() + self.join() + + def pause(self): + """ Pause/Resume download """ + self.ydl_item.pause() + +Z = 10 # sleep time + +ydl_opts = {'outtmpl': u'%(title)s.%(ext)s'} +test_url = "http://www.youtube.com/watch?v=0KSOMA3QBU0" + +dlThread = Downloader(ydl_opts, test_url) +print 'Downloading %s' % test_url +dlThread.download() +print 'Sleep %s secs' % Z +sleep(Z) +print 'Pause download for %s secs' % (Z/2) +dlThread.pause() +sleep(Z/2) +print 'Resume download' +dlThread.pause() +print 'Sleep %s secs' % Z +sleep(Z) +print 'Close downlaod thread' +dlThread.close() +print 'Downloader thread status: %s' % dlThread.isAlive() +print 'Done' From 77acafc6dd60affa1ed85913845cea9048ab5de3 Mon Sep 17 00:00:00 2001 From: MrS0m30n3 Date: Mon, 21 Apr 2014 16:17:56 +0300 Subject: [PATCH 3/5] Refactor code --- youtube_dl/YoutubeDL.py | 40 +++++++++++++-------------------- youtube_dl/downloader/common.py | 29 +++++++++++------------- youtube_dl/downloader/http.py | 6 ++--- youtube_dl/utils.py | 6 ++++- 4 files changed, 35 insertions(+), 46 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index a88df9cd0..f5a0cf7fb 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -187,8 +187,8 @@ class YoutubeDL(object): self._ies = [] self._ies_instances = {} self._pps = [] - self._stop = False - self._pause = False + self._stop_state = False + self._pause_state = False self._progress_hooks = [] self._download_retcode = 0 self._num_downloads = 0 @@ -984,8 +984,8 @@ class YoutubeDL(object): for ph in self._progress_hooks: fd.add_progress_hook(ph) # Add stop, pause handlers - fd.add_stop_handler(self._stop_handler) - fd.add_pause_handler(self._pause_handler) + fd.set_stop_handler(self.get_stop_state) + fd.set_pause_handler(self.get_pause_state) return fd.download(name, info) if info_dict.get('requested_formats') is not None: downloaded = [] @@ -1019,7 +1019,7 @@ class YoutubeDL(object): except (ContentTooShortError, ) as err: self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) return - except StopDownloads: + except StopDownloads: return if success: @@ -1031,27 +1031,17 @@ class YoutubeDL(object): self.record_download_archive(info_dict) - def _stop_handler(self): - """ Return self._stop status """ - return self._stop - - def _pause_handler(self): - """ Return self._pause status """ - return self._pause + def get_stop_state(self): + return self._stop_state - def stop(self): - """ Stop downloads """ - if self._stop: - self._stop = False - else: - self._stop = True - - def pause(self): - """ Pause/Resume downloads """ - if self._pause: - self._pause = False - else: - self._pause = True + def get_pause_state(self): + return self._pause_state + + def set_stop_state(self, stop_state): + self._stop_state = stop_state + + def set_pause_state(self, pause_state): + self._pause_state = pause_state def download(self, url_list): """Download a given list of URLs.""" diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index b8ce20af0..e16f97d5b 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -301,27 +301,24 @@ class FileDownloader(object): for ph in self._progress_hooks: ph(status) - def _stop(self): - """ Check stop handler """ + def _check_stop_handler(self): if self._stop_handler is not None: - if self._stop_handler(): - raise StopDownloads() + if self._stop_handler(): + raise StopDownloads() - def _pause(self): - """ Check pause/resume handler """ + def _check_pause_handler(self): if self._pause_handler is not None: - while self._pause_handler(): - # Break if stop handler enable - if self._stop_handler(): break - time.sleep(1) + while self._pause_handler(): + if self._stop_handler(): break + time.sleep(1) - def add_stop_handler(self, shand): - """ shand gets checked. If True raise StopDownloads """ - self._stop_handler = shand + def set_stop_handler(self, stop_handler): + """ stop_handler gets checked. If True raise StopDownloads """ + self._stop_handler = stop_handler - def add_pause_handler(self, phand): - """ phand gets checked. If True pause downloads """ - self._pause_handler = phand + def set_pause_handler(self, pause_handler): + """ pause_handler gets checked. If True pause downloads """ + self._pause_handler = pause_handler def add_progress_hook(self, ph): """ ph gets called on download progress, with a dictionary with the entries diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index 4f18c9e26..790ad9fc1 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -162,11 +162,9 @@ class HttpFD(FileDownloader): 'speed': speed, }) - # Check stop handler - self._stop() + self._check_stop_handler() - # Check pause handler - self._pause() + self._check_pause_handler() # Apply rate limit self.slow_down(start, byte_counter - resume_len) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 3daf42d99..ebd1f41a4 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -656,7 +656,11 @@ class MaxDownloadsReached(Exception): pass class StopDownloads(Exception): - """ Stop Downloads """ + """ Stop Downloads exception. + + This exception will be raised by FileDownloader objects when + YoutubDL._stop_state is True + """ pass class UnavailableVideoError(Exception): From ec4dead6b1251ccdf27f1b0a4246fc0864f010c2 Mon Sep 17 00:00:00 2001 From: MrS0m30n3 Date: Mon, 21 Apr 2014 16:19:56 +0300 Subject: [PATCH 4/5] Remove test_handlers.py --- test_handlers.py | 63 ------------------------------------------------ 1 file changed, 63 deletions(-) delete mode 100755 test_handlers.py diff --git a/test_handlers.py b/test_handlers.py deleted file mode 100755 index f2341c0b1..000000000 --- a/test_handlers.py +++ /dev/null @@ -1,63 +0,0 @@ -#! /usr/bin/env python - -# Test stop, pause & resume handlers - -from time import sleep -from threading import Thread - -from youtube_dl import YoutubeDL - -class Downloader(Thread): - - def __init__(self, options, url): - super(Downloader, self).__init__() - self.ydl_item = YoutubeDL(options) - self.url = url - - def run(self): - self.ydl_item.add_default_info_extractors() - self.ydl_item.download([self.url]) - - @staticmethod - def format_percent(db, tb): - if db is None: return None - pr = float(db) / float(tb) * 100.0 - return '%6s' % ('%3.1f%%' % pr) - - def progress_hook(self, pr): - print Downloader.format_percent(pr.get('downloaded_bytes'), pr.get('total_bytes')) - - def download(self): - """ Call self.start() """ - self.start() - - def close(self): - """ Stop download and join thread """ - self.ydl_item.stop() - self.join() - - def pause(self): - """ Pause/Resume download """ - self.ydl_item.pause() - -Z = 10 # sleep time - -ydl_opts = {'outtmpl': u'%(title)s.%(ext)s'} -test_url = "http://www.youtube.com/watch?v=0KSOMA3QBU0" - -dlThread = Downloader(ydl_opts, test_url) -print 'Downloading %s' % test_url -dlThread.download() -print 'Sleep %s secs' % Z -sleep(Z) -print 'Pause download for %s secs' % (Z/2) -dlThread.pause() -sleep(Z/2) -print 'Resume download' -dlThread.pause() -print 'Sleep %s secs' % Z -sleep(Z) -print 'Close downlaod thread' -dlThread.close() -print 'Downloader thread status: %s' % dlThread.isAlive() -print 'Done' From 9386cf9afd1345118e6af3c906c767539f8de502 Mon Sep 17 00:00:00 2001 From: MrS0m30n3 Date: Mon, 21 Apr 2014 16:34:26 +0300 Subject: [PATCH 5/5] Fix spaces Fix spaces --- youtube_dl/YoutubeDL.py | 12 ++++++------ youtube_dl/downloader/common.py | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index f5a0cf7fb..0a99616a8 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1019,8 +1019,8 @@ class YoutubeDL(object): except (ContentTooShortError, ) as err: self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) return - except StopDownloads: - return + except StopDownloads + return if success: try: @@ -1032,16 +1032,16 @@ class YoutubeDL(object): self.record_download_archive(info_dict) def get_stop_state(self): - return self._stop_state + return self._stop_state def get_pause_state(self): - return self._pause_state + return self._pause_state def set_stop_state(self, stop_state): - self._stop_state = stop_state + self._stop_state = stop_state def set_pause_state(self, pause_state): - self._pause_state = pause_state + self._pause_state = pause_state def download(self, url_list): """Download a given list of URLs.""" diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index e16f97d5b..eac7e0311 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -158,7 +158,7 @@ class FileDownloader(object): speed = float(byte_counter) / elapsed if speed > rate_limit: time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) - + def temp_name(self, filename): """Returns a temporary filename for the given filename.""" if self.params.get('nopart', False) or filename == u'-' or \ @@ -302,23 +302,23 @@ class FileDownloader(object): ph(status) def _check_stop_handler(self): - if self._stop_handler is not None: - if self._stop_handler(): - raise StopDownloads() + if self._stop_handler is not None: + if self._stop_handler(): + raise StopDownloads() def _check_pause_handler(self): - if self._pause_handler is not None: - while self._pause_handler(): - if self._stop_handler(): break - time.sleep(1) + if self._pause_handler is not None: + while self._pause_handler(): + if self._stop_handler(): break + time.sleep(1) def set_stop_handler(self, stop_handler): - """ stop_handler gets checked. If True raise StopDownloads """ - self._stop_handler = stop_handler + """ stop_handler gets checked. If True raise StopDownloads """ + self._stop_handler = stop_handler def set_pause_handler(self, pause_handler): - """ pause_handler gets checked. If True pause downloads """ - self._pause_handler = pause_handler + """ pause_handler gets checked. If True pause downloads """ + self._pause_handler = pause_handler def add_progress_hook(self, ph): """ ph gets called on download progress, with a dictionary with the entries