From 549f25758bd6a63a96fe9f7af2ff05f840fb6182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Paprota?= Date: Sat, 6 Nov 2010 14:39:40 +0100 Subject: [PATCH 1/6] Changed implementation of YoutubeUserIE to download all video ids using YouTube Data API. --- youtube-dl | 60 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/youtube-dl b/youtube-dl index b099ffd15..52bf4b823 100755 --- a/youtube-dl +++ b/youtube-dl @@ -1980,7 +1980,9 @@ class YoutubeUserIE(InfoExtractor): _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/user/(.*)' _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s' - _VIDEO_INDICATOR = r'http://gdata.youtube.com/feeds/api/videos/(.*)' # XXX Fix this. + _GDATA_PAGE_SIZE = 50 + _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d' + _VIDEO_INDICATOR = r'/watch\?v=(.+?)&' _youtube_ie = None def __init__(self, youtube_ie, downloader=None): @@ -1991,9 +1993,9 @@ class YoutubeUserIE(InfoExtractor): def suitable(url): return (re.match(YoutubeUserIE._VALID_URL, url) is not None) - def report_download_page(self, username): + def report_download_page(self, username, start_index): """Report attempt to download user page.""" - self._downloader.to_screen(u'[youtube] user %s: Downloading page ' % (username)) + self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' % (username, start_index, start_index + self._GDATA_PAGE_SIZE)) def _real_initialize(self): self._youtube_ie.initialize() @@ -2005,34 +2007,44 @@ class YoutubeUserIE(InfoExtractor): self._downloader.trouble(u'ERROR: invalid url: %s' % url) return - # Download user page username = mobj.group(1) + + # Download video ids using YouTube Data API. Result size per query is limited (currently to 50 videos) so + # we need to query page by page until there are no video ids - it means we got all of them. + video_ids = [] - pagenum = 1 + pagenum = 0 - self.report_download_page(username) - request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers) - try: - page = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) - return + while True: + start_index = pagenum * self._GDATA_PAGE_SIZE + 1 + self.report_download_page(username, start_index) - # Extract video identifiers - ids_in_page = [] + request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index), None, std_headers) - for mobj in re.finditer(self._VIDEO_INDICATOR, page): - if mobj.group(1) not in ids_in_page: - ids_in_page.append(mobj.group(1)) - video_ids.extend(ids_in_page) + try: + page = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) + return - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - video_ids = video_ids[playliststart:playlistend] + # Extract video identifiers + ids_in_page = [] - for id in video_ids: - self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id) - return + for mobj in re.finditer(self._VIDEO_INDICATOR, page): + if mobj.group(1) not in ids_in_page: + ids_in_page.append(mobj.group(1)) + + if len(ids_in_page) == 0: + # Got them all, now download! + break + + pagenum += 1 + video_ids.extend(ids_in_page) + + self._downloader.to_screen("[youtube] user %s: Collected %d video ids" % (username, len(video_ids))) + + for video_id in video_ids: + self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id) class PostProcessor(object): """Post Processor class. From afef7c15994e73da7272bcfa6ed761405824f106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Paprota?= Date: Sat, 11 Dec 2010 17:50:20 +0100 Subject: [PATCH 2/6] Ignore download errors because they can interrupt downloading all user videos. --- youtube-dl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube-dl b/youtube-dl index 52bf4b823..ab6dcb027 100755 --- a/youtube-dl +++ b/youtube-dl @@ -2044,7 +2044,10 @@ class YoutubeUserIE(InfoExtractor): self._downloader.to_screen("[youtube] user %s: Collected %d video ids" % (username, len(video_ids))) for video_id in video_ids: - self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id) + try: + self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id) + except DownloadError: + continue class PostProcessor(object): """Post Processor class. From a51c00f70c24a2d41bbf963cb2ba96da3395e068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Paprota?= Date: Thu, 23 Dec 2010 09:08:43 +0100 Subject: [PATCH 3/6] Be smarter about downloading pages of user video ids - now no extra page is downloaded when we have all ids. --- youtube-dl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/youtube-dl b/youtube-dl index 73fc45827..dcbecd336 100755 --- a/youtube-dl +++ b/youtube-dl @@ -33,7 +33,7 @@ std_headers = { 'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.12) Gecko/20101028 Firefox/3.6.12', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Encoding': 'gzip, deflate', +# 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-us,en;q=0.5', } @@ -2086,12 +2086,15 @@ class YoutubeUserIE(InfoExtractor): if mobj.group(1) not in ids_in_page: ids_in_page.append(mobj.group(1)) - if len(ids_in_page) == 0: - # Got them all, now download! + video_ids.extend(ids_in_page) + + # A little optimization - if current page is not "full", ie. does not contain PAGE_SIZE video ids then we can assume + # that this page is the last one - there are no more ids on further pages - no need to query again. + + if len(ids_in_page) < self._GDATA_PAGE_SIZE: break pagenum += 1 - video_ids.extend(ids_in_page) self._downloader.to_screen("[youtube] user %s: Collected %d video ids" % (username, len(video_ids))) From dc093a4babbba8dd69c316a9e37217bd97bad1d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Paprota?= Date: Thu, 27 Jan 2011 10:32:23 +0100 Subject: [PATCH 4/6] Commented out 'Accept-Encoding' when it was not handled properly, now it seems to work fine after recent commits on master - so uncommenting it. --- youtube-dl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube-dl b/youtube-dl index 19c327e97..0c746ae6e 100755 --- a/youtube-dl +++ b/youtube-dl @@ -38,7 +38,7 @@ std_headers = { 'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.12) Gecko/20101028 Firefox/3.6.12', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', -# 'Accept-Encoding': 'gzip, deflate', + 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-us,en;q=0.5', } From 6608d7fe620195397cda088ab99ddb4db9ab8be1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Paprota?= Date: Fri, 28 Jan 2011 10:19:56 +0100 Subject: [PATCH 5/6] Added support for ytuser:USERNAME argument format. --- youtube-dl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube-dl b/youtube-dl index 0c746ae6e..6ecf04615 100755 --- a/youtube-dl +++ b/youtube-dl @@ -2117,7 +2117,7 @@ class YoutubePlaylistIE(InfoExtractor): class YoutubeUserIE(InfoExtractor): """Information Extractor for YouTube users.""" - _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/user/(.*)' + _VALID_URL = r'(?:(?:http://)?(?:\w+\.)?youtube.com/user/(.*)|ytuser:([^\s]+))' _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s' _GDATA_PAGE_SIZE = 50 _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d' @@ -2148,6 +2148,10 @@ class YoutubeUserIE(InfoExtractor): username = mobj.group(1) + if not username: + # Probably the second group matched - meaning that the argument in the format "ytuser:USERNAME" was used. + username = mobj.group(2) + # Download video ids using YouTube Data API. Result size per query is limited (currently to 50 videos) so # we need to query page by page until there are no video ids - it means we got all of them. From fd174ae83d5ad962ccbf5639c360943ac942b22e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Paprota?= Date: Fri, 28 Jan 2011 10:29:20 +0100 Subject: [PATCH 6/6] Added support for playlist start/end parameters. --- youtube-dl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/youtube-dl b/youtube-dl index 6ecf04615..8552b9988 100755 --- a/youtube-dl +++ b/youtube-dl @@ -2187,7 +2187,16 @@ class YoutubeUserIE(InfoExtractor): pagenum += 1 - self._downloader.to_screen("[youtube] user %s: Collected %d video ids" % (username, len(video_ids))) + all_ids_count = len(video_ids) + playliststart = self._downloader.params.get('playliststart', 1) - 1 + playlistend = self._downloader.params.get('playlistend', -1) + + if playlistend == -1: + video_ids = video_ids[playliststart:] + else: + video_ids = video_ids[playliststart:playlistend] + + self._downloader.to_screen("[youtube] user %s: Collected %d video ids (downloading %d of them)" % (username, all_ids_count, len(video_ids))) for video_id in video_ids: try: