From c7667c2d7f602aecfd8a39f26d8151a363ba0b5e Mon Sep 17 00:00:00 2001
From: SyxbEaEQ2 <SyxbEaEQ2@users.noreply.github.com>
Date: Thu, 31 Jul 2014 03:08:24 +0200
Subject: [PATCH 001/974] [downloader/(common/http)] Changes calculation of the
 rate-limit. (Fix #2297, fix #2140, fix #595, fix #2370)

---
 youtube_dl/downloader/common.py | 15 +++++++++------
 youtube_dl/downloader/http.py   | 31 ++++++++++++++++++++++---------
 2 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 917f3450e..6404e1928 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -77,8 +77,10 @@ class FileDownloader(object):
     def calc_eta(start, now, total, current):
         if total is None:
             return None
+        if now is None:
+            now = time.time()
         dif = now - start
-        if current == 0 or dif < 0.001: # One millisecond
+        if current == 0 or dif < 0.001:  # One millisecond
             return None
         rate = float(current) / dif
         return int((float(total) - float(current)) / rate)
@@ -92,7 +94,7 @@ class FileDownloader(object):
     @staticmethod
     def calc_speed(start, now, bytes):
         dif = now - start
-        if bytes == 0 or dif < 0.001: # One millisecond
+        if bytes == 0 or dif < 0.001:  # One millisecond
             return None
         return float(bytes) / dif
 
@@ -105,7 +107,7 @@ class FileDownloader(object):
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
-        new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
+        new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
         if elapsed_time < 0.001:
             return int(new_max)
         rate = bytes / elapsed_time
@@ -143,18 +145,19 @@ class FileDownloader(object):
     def report_error(self, *args, **kargs):
         self.ydl.report_error(*args, **kargs)
 
-    def slow_down(self, start_time, byte_counter):
+    def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""
         rate_limit = self.params.get('ratelimit', None)
         if rate_limit is None or byte_counter == 0:
             return
-        now = time.time()
+        if now is None:
+            now = time.time()
         elapsed = now - start_time
         if elapsed <= 0.0:
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:
-            time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
+            time.sleep((byte_counter / rate_limit) - elapsed)
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index f79e6a995..462be2739 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -128,16 +128,21 @@ class HttpFD(FileDownloader):
         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)
         start = time.time()
+
+        # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
+        now = None  # needed for slow_down() in the first loop run
+        before = start  # start measuring
         while True:
+
             # Download and write
-            before = time.time()
             data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
-            after = time.time()
-            if len(data_block) == 0:
-                break
             byte_counter += len(data_block)
 
-            # Open file just in time
+            # exit loop when download is finished
+            if len(data_block) == 0:
+                break
+
+            # Open destination file just in time
             if stream is None:
                 try:
                     (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
@@ -153,11 +158,22 @@ class HttpFD(FileDownloader):
                 self.to_stderr(u"\n")
                 self.report_error(u'unable to write data: %s' % str(err))
                 return False
+
+            # Apply rate limit
+            self.slow_down(start, now, byte_counter - resume_len)
+
+            # end measuring of one loop run
+            now = time.time()
+            after = now
+
+            # Adjust block size
             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
 
+            before = after
+
             # Progress message
-            speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
+            speed = self.calc_speed(start, now, byte_counter - resume_len)
             if data_len is None:
                 eta = percent = None
             else:
@@ -178,9 +194,6 @@ class HttpFD(FileDownloader):
             if is_test and byte_counter == data_len:
                 break
 
-            # Apply rate limit
-            self.slow_down(start, byte_counter - resume_len)
-
         if stream is None:
             self.to_stderr(u"\n")
             self.report_error(u'Did not get any data blocks')

From 03359e9864bfb925f577fa5b16c3ef22884127aa Mon Sep 17 00:00:00 2001
From: rupertbaxter2 <rupertbaxter2@yahoo.com>
Date: Sun, 3 Aug 2014 07:34:04 -0700
Subject: [PATCH 002/974] Added --sleep-interval option

---
 youtube_dl/__init__.py          | 8 ++++++++
 youtube_dl/downloader/common.py | 3 +++
 youtube_dl/utils.py             | 3 +++
 3 files changed, 14 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 429630ce5..2bd5ec33b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -351,6 +351,8 @@ def parseOpts(overrideArguments=None):
 
     downloader.add_option('-r', '--rate-limit',
             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
+    downloader.add_option('--sleep-interval',
+            dest='sleepinterval', metavar='SLEEPINTERVAL', help='number of seconds to sleep between downloads (default is %default)', default="0")
     downloader.add_option('-R', '--retries',
             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
     downloader.add_option('--buffer-size',
@@ -671,6 +673,11 @@ def _real_main(argv=None):
         if numeric_limit is None:
             parser.error(u'invalid rate limit specified')
         opts.ratelimit = numeric_limit
+    if opts.sleepinterval is not None:
+        try:
+            opts.sleepinterval = abs(int(opts.sleepinterval))
+        except ValueError:
+            parser.error(u'invalid sleep interval specified')
     if opts.min_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
         if numeric_limit is None:
@@ -767,6 +774,7 @@ def _real_main(argv=None):
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
         'ratelimit': opts.ratelimit,
+        'sleepinterval': opts.sleepinterval,
         'nooverwrites': opts.nooverwrites,
         'retries': opts.retries,
         'buffersize': opts.buffersize,
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 917f3450e..8e0e386bf 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -278,6 +278,9 @@ class FileDownloader(object):
         """Download to a filename using the info from info_dict
         Return True on success and False otherwise
         """
+        sleep_interval = self.params.get('sleepinterval', 0)
+        self.to_screen(u'[download] Sleeping %d seconds...' %sleep_interval)
+        time.sleep(sleep_interval)
         # Check file already present
         if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
             self.report_file_already_downloaded(filename)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e40b367c2..d199d26d2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -6,6 +6,7 @@ import codecs
 import contextlib
 import ctypes
 import datetime
+import time
 import email.utils
 import errno
 import getpass
@@ -747,6 +748,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 del req.headers['User-agent']
             req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
             del req.headers['Youtubedl-user-agent']
+        #print("sleeping\n")
+        #time.sleep(1)
         return req
 
     def http_response(self, req, resp):

From 2f61fe4cccc1ef4186943f4eed2e89f8fe2e2c23 Mon Sep 17 00:00:00 2001
From: rupertbaxter2 <rupertbaxter2@yahoo.com>
Date: Sun, 3 Aug 2014 07:38:04 -0700
Subject: [PATCH 003/974] Removed unneccesary changes to utils.py

---
 youtube_dl/utils.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d199d26d2..e40b367c2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -6,7 +6,6 @@ import codecs
 import contextlib
 import ctypes
 import datetime
-import time
 import email.utils
 import errno
 import getpass
@@ -748,8 +747,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 del req.headers['User-agent']
             req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
             del req.headers['Youtubedl-user-agent']
-        #print("sleeping\n")
-        #time.sleep(1)
         return req
 
     def http_response(self, req, resp):

From 00cf122d7a79e81a2b328b7352d23eb0bdb17e52 Mon Sep 17 00:00:00 2001
From: SyxbEaEQ2 <SyxbEaEQ2@users.noreply.github.com>
Date: Wed, 6 Aug 2014 20:53:04 +0200
Subject: [PATCH 004/974]  [downloader/common] Fix possible negative sleep time
 in slow_down()

---
 youtube_dl/downloader/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 6404e1928..33ebbf6b4 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -157,7 +157,7 @@ class FileDownloader(object):
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:
-            time.sleep((byte_counter / rate_limit) - elapsed)
+            time.sleep(max((byte_counter / rate_limit) - elapsed, 0))
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""

From a42c9215983c4d62d1c000c9dede6e0850dbb5e4 Mon Sep 17 00:00:00 2001
From: rupertbaxter2 <rupertbaxter2@yahoo.com>
Date: Wed, 13 Aug 2014 04:38:40 -0700
Subject: [PATCH 005/974] Removed sleep and sleep output when interval is zero

---
 youtube_dl/downloader/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 48e829deb..c1da065b5 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -279,8 +279,9 @@ class FileDownloader(object):
         Return True on success and False otherwise
         """
         sleep_interval = self.params.get('sleepinterval', 0)
-        self.to_screen(u'[download] Sleeping %d seconds...' %sleep_interval)
-        time.sleep(sleep_interval)
+        if sleep_interval > 0:
+            self.to_screen(u'[download] Sleeping %d seconds...' %sleep_interval)
+            time.sleep(sleep_interval)
         # Check file already present
         if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
             self.report_file_already_downloaded(filename)

From c816336cbdb91efa282c0ede8552157861f10e76 Mon Sep 17 00:00:00 2001
From: net <net@h-MacBook-Pro-sl-net.local>
Date: Mon, 29 Sep 2014 21:58:42 +0300
Subject: [PATCH 006/974] [karaoketv] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/karaoketv.py | 47 +++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 youtube_dl/extractor/karaoketv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 079221567..d12e741a0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -173,6 +173,7 @@ from .jukebox import JukeboxIE
 from .justintv import JustinTVIE
 from .jpopsukitv import JpopsukiIE
 from .kankan import KankanIE
+from .karaoketv import KaraoketvIE
 from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
new file mode 100644
index 000000000..4d50308cc
--- /dev/null
+++ b/youtube_dl/extractor/karaoketv.py
@@ -0,0 +1,47 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+import sys
+
+from .common import InfoExtractor
+from ..utils import compat_urllib_parse, ExtractorError
+
+
+class KaraoketvIE(InfoExtractor):
+    _VALID_URL = r'http://karaoketv\.co\.il/\?container=songs&id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://karaoketv.co.il/?container=songs&id=171568',
+        'info_dict': {
+            'id': '171568',
+            'ext': 'mp4',
+            'title': 'אל העולם שלך - רותם כהן - שרים קריוקי',
+        }
+    }
+
+    def _real_extract(self, url):
+
+        # BUG: SSL23_GET_SERVER_HELLO:unknown protocol 
+        if sys.hexversion < 0x03000000:
+            raise ExtractorError("Only python 3 supported.\n")
+
+        mobj = re.match(self._VALID_URL, url)
+        
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        settings_json = compat_urllib_parse.unquote_plus(self._search_regex(r'config=(.*)', self._og_search_video_url(webpage ,video_id), ''))
+        
+        urls_info_webpage = self._download_webpage(settings_json, 'Downloading settings json')
+
+        urls_info_json = json.loads(urls_info_webpage.replace('\'', '"'))
+
+        url = urls_info_json['playlist'][0]['url']
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'url': url,
+        }
\ No newline at end of file

From b1c3a49fffb7109125a2ad215f412f1198e3dffd Mon Sep 17 00:00:00 2001
From: "Ching Yi, Chan" <chingyichan.tw@gmail.com>
Date: Sun, 12 Oct 2014 08:32:26 +0800
Subject: [PATCH 007/974] apply ratelimit to f4m

---
 youtube_dl/downloader/f4m.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b3be16ff1..54dd6ac3f 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -230,6 +230,7 @@ class F4mFD(FileDownloader):
                 'continuedl': True,
                 'quiet': True,
                 'noprogress': True,
+                'ratelimit': self.params.get('ratelimit', None),
                 'test': self.params.get('test', False),
             })
 

From 7b61ac3ddf0bfe660266e70fc7c10f048758698f Mon Sep 17 00:00:00 2001
From: net <net@h-MacBook-Pro-sl-net.local>
Date: Wed, 15 Oct 2014 06:46:47 +0300
Subject: [PATCH 008/974] Fix #2310. Play by the 8tracks rules

---
 youtube_dl/extractor/eighttracks.py | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index c1b4c729e..101a09699 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -4,10 +4,12 @@ from __future__ import unicode_literals
 import json
 import random
 import re
+import time
 
 from .common import InfoExtractor
 from ..utils import (
     compat_str,
+    ExtractorError,
 )
 
 
@@ -112,14 +114,30 @@ class EightTracksIE(InfoExtractor):
         session = str(random.randint(0, 1000000000))
         mix_id = data['id']
         track_count = data['tracks_count']
+        duration = data['duration']
+        avg_song_duration = duration / track_count
         first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
         next_url = first_url
         entries = []
+
         for i in range(track_count):
-            api_json = self._download_webpage(
-                next_url, playlist_id,
-                note='Downloading song information %d/%d' % (i + 1, track_count),
-                errnote='Failed to download song information')
+            
+            api_json = None
+            download_tries = 0
+
+            while api_json is None:
+                try:
+                    api_json = self._download_webpage(
+                        next_url, playlist_id,
+                        note='Downloading song information %d/%d' % (i + 1, track_count),
+                        errnote='Failed to download song information')
+                except ExtractorError:
+                    if download_tries > 3:
+                        raise
+                    else:
+                        ++download_tries
+                        time.sleep(avg_song_duration)
+
             api_data = json.loads(api_json)
             track_data = api_data['set']['track']
             info = {
@@ -131,6 +149,7 @@ class EightTracksIE(InfoExtractor):
                 'ext': 'm4a',
             }
             entries.append(info)
+
             next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (
                 session, mix_id, track_data['id'])
         return {

From 4231235cdaa18d050f3119a80ac409138fb8e8bd Mon Sep 17 00:00:00 2001
From: h-collector <githcoll@gmail.com>
Date: Sun, 6 Jul 2014 01:42:41 +0200
Subject: [PATCH 009/974] Fix issues with fc2

Fix issues #2912 and #3171
---
 youtube_dl/extractor/fc2.py | 49 ++++++++++++++++++++++++++++++++++---
 1 file changed, 46 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index c663a0f81..ecfb233f9 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -7,14 +7,16 @@ import hashlib
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
 )
 
 
 class FC2IE(InfoExtractor):
-    _VALID_URL = r'^http://video\.fc2\.com/((?P<lang>[^/]+)/)?content/(?P<id>[^/]+)'
+    _VALID_URL = r'^http://video\.fc2\.com/((?P<lang>[^/]+)/)?(a/)?content/(?P<id>[^/]+)'
     IE_NAME = 'fc2'
+    _NETRC_MACHINE = 'fc2'
     _TEST = {
         'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
         'md5': 'a6ebe8ebe0396518689d963774a54eb7',
@@ -25,17 +27,53 @@ class FC2IE(InfoExtractor):
         },
     }
 
+    #def _real_initialize(self):
+    #    self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if (username is None) or (password is None):
+           self._downloader.report_warning('unable to log in: will be downloading in non authorized mode')
+           return False
+
+        # Log in
+        login_form_strs = {
+            'email':    username,
+            'password': password,
+            'done':     'video',
+            'Submit':   ' Login ',
+        }
+
+        # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
+        # chokes on unicode
+        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
+        login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8')
+        request    = compat_urllib_request.Request(
+            'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
+
+        login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
+        if 'mode=redirect&login=done' not in login_results:
+            self._downloader.report_warning('unable to log in: bad username or password')
+            return False
+        
+        # this is also needed
+        login_redir = compat_urllib_request.Request('http://id.fc2.com/?mode=redirect&login=done')
+        redir_res   = self._download_webpage(login_redir, None, note='Login redirect', errnote='Something is not right')
+
+        return True
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id)
         self._downloader.cookiejar.clear_session_cookies()  # must clear
+        self._login()
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        refer = url.replace('/content/', '/a/content/')
 
+        refer = (url if '/a/content/' in url else url.replace('/content/', '/a/content/'));
         mimi = hashlib.md5((video_id + '_gGddgPfeaf_gzyr').encode('utf-8')).hexdigest()
 
         info_url = (
@@ -47,7 +85,12 @@ class FC2IE(InfoExtractor):
         info = compat_urlparse.parse_qs(info_webpage)
 
         if 'err_code' in info:
-            raise ExtractorError('Error code: %s' % info['err_code'][0])
+            #raise ExtractorError('Error code: %s' % info['err_code'][0])
+            # most of the time we can still download wideo even if err_code is 403 or 602
+            print 'Error code was: %s... but still trying' % info['err_code'][0]
+            
+        if 'filepath' not in info:
+            raise ExtractorError('No file path for download. Maybe not logged?')
 
         video_url = info['filepath'][0] + '?mid=' + info['mid'][0]
         title_info = info.get('title')

From 40b1cbafacea338e51d43fa78438eaf21a1bcbcd Mon Sep 17 00:00:00 2001
From: h-collector <githcoll@gmail.com>
Date: Sun, 6 Jul 2014 01:48:07 +0200
Subject: [PATCH 010/974] Update fc2.py

---
 youtube_dl/extractor/fc2.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index ecfb233f9..0933485e0 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -87,10 +87,10 @@ class FC2IE(InfoExtractor):
         if 'err_code' in info:
             #raise ExtractorError('Error code: %s' % info['err_code'][0])
             # most of the time we can still download wideo even if err_code is 403 or 602
-            print 'Error code was: %s... but still trying' % info['err_code'][0]
-            
+            self._downloader.report_warning('Error code was: %s... but still trying' % info['err_code'][0])
+
         if 'filepath' not in info:
-            raise ExtractorError('No file path for download. Maybe not logged?')
+            raise ExtractorError('Cannot download file. Are you logged?')
 
         video_url = info['filepath'][0] + '?mid=' + info['mid'][0]
         title_info = info.get('title')

From 5a000b45b339f6516f2a5a3bdfd2869713e8438a Mon Sep 17 00:00:00 2001
From: h-collector <githcoll@gmail.com>
Date: Mon, 7 Jul 2014 22:47:28 +0200
Subject: [PATCH 011/974] Don't use report_warning for reporting warnings

In tests warning is converted to error
---
 youtube_dl/extractor/fc2.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 0933485e0..3073ab0d4 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -33,7 +33,7 @@ class FC2IE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if (username is None) or (password is None):
-           self._downloader.report_warning('unable to log in: will be downloading in non authorized mode')
+           self.to_screen('unable to log in: will be downloading in non authorized mode') # report_warning
            return False
 
         # Log in
@@ -53,7 +53,7 @@ class FC2IE(InfoExtractor):
 
         login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
         if 'mode=redirect&login=done' not in login_results:
-            self._downloader.report_warning('unable to log in: bad username or password')
+            self.to_screen('unable to log in: bad username or password') # report_warning
             return False
         
         # this is also needed
@@ -87,7 +87,7 @@ class FC2IE(InfoExtractor):
         if 'err_code' in info:
             #raise ExtractorError('Error code: %s' % info['err_code'][0])
             # most of the time we can still download wideo even if err_code is 403 or 602
-            self._downloader.report_warning('Error code was: %s... but still trying' % info['err_code'][0])
+            self.to_screen('Error code was: %s... but still trying' % info['err_code'][0]) # report_warning
 
         if 'filepath' not in info:
             raise ExtractorError('Cannot download file. Are you logged?')

From 34e48bed3b562ef32c3bfa14ba59101786316464 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Fri, 24 Oct 2014 02:41:10 +0530
Subject: [PATCH 012/974] [extractor/common] Added support for f4m manifest
 Version 2.0

---
 youtube_dl/extractor/common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e8366f7f9..290df1175 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -648,6 +648,8 @@ class InfoExtractor(object):
 
         formats = []
         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
+        if not media_nodes:
+            media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             format_id = 'f4m-%d' % (i if tbr is None else tbr)

From 7a47d07c6dbc8b6bb3be321af886c1774d68e0d6 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Fri, 24 Oct 2014 09:47:39 +0530
Subject: [PATCH 013/974] [extractor/common] href attribute added

---
 youtube_dl/extractor/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 290df1175..a8674dd85 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -647,10 +647,14 @@ class InfoExtractor(object):
             'Unable to download f4m manifest')
 
         formats = []
+        manifest_version = '1.0'
         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
         if not media_nodes:
+            manifest_version = '2.0'
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
+            if manifest_version == '2.0':
+                manifest_url = '/'.join(manifest_url.split('/')[:-1]) + '/' + media_el.attrib.get('href')
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             format_id = 'f4m-%d' % (i if tbr is None else tbr)
             formats.append({

From 25fadd06d078d54e30e187063d8b45fae31bed81 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Fri, 24 Oct 2014 09:49:01 +0530
Subject: [PATCH 014/974] [Rte] New extractor added

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rte.py      | 53 ++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/rte.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0de0b4587..7918f9ab7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -297,6 +297,7 @@ from .ro220 import Ro220IE
 from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
+from .rte import RteIE
 from .rtlnl import RtlXlIE
 from .rtlnow import RTLnowIE
 from .rts import RTSIE
diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
new file mode 100644
index 000000000..ab12bb5fd
--- /dev/null
+++ b/youtube_dl/extractor/rte.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    float_or_none,
+)
+
+
+class RteIE(InfoExtractor):
+    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/in/show/(?P<id>[0-9]+)/'
+    _TEST = {
+        'url': 'http://www.rte.ie/player/in/show/10336191/',
+        'info_dict': {
+            'id': '10336191',
+            'ext': 'mp4',
+            'title': 'Nine News',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'The One O\'Clock News followed by Weather.',
+            'duration': 1622963.0,
+        }
+    }
+    
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._og_search_title(webpage)
+        
+        description = self._search_regex(r'<meta name="description" content="(.*?)" />', webpage, 'description')
+        duration = float_or_none(self._html_search_meta('duration', webpage, 'duration'))
+        
+        thumbnail_id = self._search_regex(r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
+        thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg' 
+        
+        feeds_url = self._html_search_meta("feeds-prefix", webpage, 'feeds url') + video_id
+        json_string = self._download_json(feeds_url, video_id)
+        
+        # f4m_url = server + relative_url
+        f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url']
+        f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
+        for f in f4m_formats:
+            del f['tbr']
+        
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': f4m_formats,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }
\ No newline at end of file

From 0551a02b82b4209422e0734150e0186210cd0723 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Mon, 27 Oct 2014 01:08:51 +0530
Subject: [PATCH 015/974] [Rte] Improve extractor

---
 youtube_dl/extractor/rte.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index ab12bb5fd..8d4e94e55 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -18,7 +18,7 @@ class RteIE(InfoExtractor):
             'title': 'Nine News',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'The One O\'Clock News followed by Weather.',
-            'duration': 1622963.0,
+            'duration': 1622.963,
         }
     }
     
@@ -29,7 +29,7 @@ class RteIE(InfoExtractor):
         title = self._og_search_title(webpage)
         
         description = self._search_regex(r'<meta name="description" content="(.*?)" />', webpage, 'description')
-        duration = float_or_none(self._html_search_meta('duration', webpage, 'duration'))
+        duration = float_or_none(self._html_search_meta('duration', webpage, 'duration'), 1000)
         
         thumbnail_id = self._search_regex(r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
         thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg' 
@@ -40,8 +40,13 @@ class RteIE(InfoExtractor):
         # f4m_url = server + relative_url
         f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url']
         f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
-        for f in f4m_formats:
-            del f['tbr']
+        f4m_formats = [{
+            'format_id': f['format_id'],
+            'url': f['url'],
+            'ext': 'mp4',
+            'width': f['width'],
+            'height': f['height'],
+        } for f in f4m_formats ]
         
         return {
             'id': video_id,

From 0cd64bd077fb378ae3afc5b2a8e091e4f50df9c2 Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Thu, 20 Nov 2014 06:25:12 +0000
Subject: [PATCH 016/974] Copy the mtime from the oldest source file to the
 file created by ffmpeg

Fixes #4245
---
 youtube_dl/postprocessor/ffmpeg.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index f3f2743c0..d44de84cf 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -79,6 +79,7 @@ class FFmpegPostProcessor(PostProcessor):
         files_cmd = []
         for path in input_paths:
             files_cmd.extend(['-i', encodeFilename(path, True)])
+        oldest_mtime = min(os.stat(path).st_mtime for path in input_paths)
         cmd = ([self._executable, '-y'] + files_cmd
                + [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
@@ -91,6 +92,7 @@ class FFmpegPostProcessor(PostProcessor):
             stderr = stderr.decode('utf-8', 'replace')
             msg = stderr.strip().split('\n')[-1]
             raise FFmpegPostProcessorError(msg)
+        os.utime(out_path, (oldest_mtime, oldest_mtime))
         if self._deletetempfiles:
             for ipath in input_paths:
                 os.remove(ipath)

From 313d4572ce782a639e42b454edadc0b24d7799a1 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Sun, 30 Nov 2014 16:04:04 +0000
Subject: [PATCH 017/974] [xminus] update tkn extraction regex

---
 youtube_dl/extractor/xminus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xminus.py b/youtube_dl/extractor/xminus.py
index a3fd8f48c..f7e2e8ac9 100644
--- a/youtube_dl/extractor/xminus.py
+++ b/youtube_dl/extractor/xminus.py
@@ -50,7 +50,7 @@ class XMinusIE(InfoExtractor):
             webpage, 'view count', fatal=False))
 
         enc_token = self._html_search_regex(
-            r'data-mt="(.*?)"', webpage, 'enc_token')
+            r'minus_track\.tkn="(.+?)"', webpage, 'enc_token')
         token = ''.join(
             c if pos == 3 else compat_chr(compat_ord(c) - 1)
             for pos, c in enumerate(reversed(enc_token)))

From c056efa2e3e450ef70f0b23559273e2ec0f4e218 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Nov 2014 22:37:56 +0600
Subject: [PATCH 018/974] [bbccouk] Fix extraction (#4104, #4214)

---
 youtube_dl/extractor/bbccouk.py | 122 +++++++++++++++++++++-----------
 1 file changed, 81 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 6a507e113..beb6cfc8a 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -1,9 +1,11 @@
 from __future__ import unicode_literals
 
 import re
+import xml.etree.ElementTree
 
 from .subtitles import SubtitlesInfoExtractor
 from ..utils import ExtractorError
+from ..compat import compat_HTTPError
 
 
 class BBCCoUkIE(SubtitlesInfoExtractor):
@@ -55,7 +57,22 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
                 'skip_download': True,
             },
             'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
-        }
+        },
+        {
+            'url': 'http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion',
+            'info_dict': {
+                'id': 'b03k3pb7',
+                'ext': 'flv',
+                'title': "Tomorrow's Worlds: The Unearthly History of Science Fiction",
+                'description': '2. Invasion',
+                'duration': 3600,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+            'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
+        },
     ]
 
     def _extract_asx_playlist(self, connection, programme_id):
@@ -102,6 +119,10 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         return playlist.findall('./{http://bbc.co.uk/2008/emp/playlist}item')
 
     def _extract_medias(self, media_selection):
+        error = media_selection.find('./{http://bbc.co.uk/2008/mp/mediaselection}error')
+        if error is not None:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error.get('id')), expected=True)
         return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')
 
     def _extract_connections(self, media):
@@ -158,54 +179,73 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             subtitles[lang] = srt
         return subtitles
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        group_id = mobj.group('id')
-
-        webpage = self._download_webpage(url, group_id, 'Downloading video page')
-        if re.search(r'id="emp-error" class="notinuk">', webpage):
-            raise ExtractorError('Currently BBC iPlayer TV programmes are available to play in the UK only',
-                                 expected=True)
-
-        playlist = self._download_xml('http://www.bbc.co.uk/iplayer/playlist/%s' % group_id, group_id,
-                                      'Downloading playlist XML')
-
-        no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
-        if no_items is not None:
-            reason = no_items.get('reason')
-            if reason == 'preAvailability':
-                msg = 'Episode %s is not yet available' % group_id
-            elif reason == 'postAvailability':
-                msg = 'Episode %s is no longer available' % group_id
+    def _download_media_selector(self, programme_id):
+        try:
+            media_selection = self._download_xml(
+                'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' % programme_id,
+                programme_id, 'Downloading media selection XML')
+        except ExtractorError as ee:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
             else:
-                msg = 'Episode %s is not available: %s' % (group_id, reason)
-            raise ExtractorError(msg, expected=True)
+                raise
 
         formats = []
         subtitles = None
 
-        for item in self._extract_items(playlist):
-            kind = item.get('kind')
-            if kind != 'programme' and kind != 'radioProgramme':
-                continue
-            title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
-            description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
+        for media in self._extract_medias(media_selection):
+            kind = media.get('kind')
+            if kind == 'audio':
+                formats.extend(self._extract_audio(media, programme_id))
+            elif kind == 'video':
+                formats.extend(self._extract_video(media, programme_id))
+            elif kind == 'captions':
+                subtitles = self._extract_captions(media, programme_id)
 
-            programme_id = item.get('identifier')
-            duration = int(item.get('duration'))
+        return formats, subtitles
 
-            media_selection = self._download_xml(
-                'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' % programme_id,
-                programme_id, 'Downloading media selection XML')
+    def _real_extract(self, url):
+        group_id = self._match_id(url)
 
-            for media in self._extract_medias(media_selection):
-                kind = media.get('kind')
-                if kind == 'audio':
-                    formats.extend(self._extract_audio(media, programme_id))
-                elif kind == 'video':
-                    formats.extend(self._extract_video(media, programme_id))
-                elif kind == 'captions':
-                    subtitles = self._extract_captions(media, programme_id)
+        webpage = self._download_webpage(url, group_id, 'Downloading video page')
+
+        programme_id = self._search_regex(
+            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False)
+        if programme_id:
+            player = self._download_json(
+                'http://www.bbc.co.uk/iplayer/episode/%s.json' % group_id,
+                group_id)['jsConf']['player']
+            title = player['title']
+            description = player['subtitle']
+            duration = player['duration']
+            formats, subtitles = self._download_media_selector(programme_id)
+        else:
+            playlist = self._download_xml(
+                'http://www.bbc.co.uk/iplayer/playlist/%s' % group_id,
+                group_id, 'Downloading playlist XML')
+
+            no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
+            if no_items is not None:
+                reason = no_items.get('reason')
+                if reason == 'preAvailability':
+                    msg = 'Episode %s is not yet available' % group_id
+                elif reason == 'postAvailability':
+                    msg = 'Episode %s is no longer available' % group_id
+                elif reason == 'noMedia':
+                    msg = 'Episode %s is not currently available' % group_id
+                else:
+                    msg = 'Episode %s is not available: %s' % (group_id, reason)
+                raise ExtractorError(msg, expected=True)
+
+            for item in self._extract_items(playlist):
+                kind = item.get('kind')
+                if kind != 'programme' and kind != 'radioProgramme':
+                    continue
+                title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
+                description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
+                programme_id = item.get('identifier')
+                duration = int(item.get('duration'))
+                formats, subtitles = self._download_media_selector(programme_id)
 
         if self._downloader.params.get('listsubtitles', False):
             self._list_available_subtitles(programme_id, subtitles)

From 10a404c335295d76678560eedaa5520c2aafae6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 18:56:14 +0100
Subject: [PATCH 019/974] [youtube] Add format 313 (fixes #4339)

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 96b49fed1..eaa5651fe 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -285,6 +285,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'VP9'},
 
         # Dash webm audio
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},

From b766eb2707c5fcd0ba55c1bc2a5d3e98aaa3b2c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 19:18:39 +0100
Subject: [PATCH 020/974] [youtube] Update test

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eaa5651fe..4b19061a9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -379,8 +379,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'info_dict': {
                 'id': 'IB3lcPjvWLA',
                 'ext': 'm4a',
-                'title': 'Afrojack - The Spark ft. Spree Wilson',
-                'description': 'md5:9717375db5a9a3992be4668bbf3bc0a8',
+                'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
+                'description': 'md5:12e7067fa6735a77bdcbb58cb1187d2d',
                 'uploader': 'AfrojackVEVO',
                 'uploader_id': 'AfrojackVEVO',
                 'upload_date': '20131011',

From 4472f84f0c09eba9720e3447fc8c0b7b19a8c20e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 19:42:54 +0100
Subject: [PATCH 021/974] [test/test_subtitles] Update checksum for vimeo
 subtitle file

---
 test/test_subtitles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 2eeb31bc6..7c4cd8218 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -238,7 +238,7 @@ class TestVimeoSubtitles(BaseTestSubtitles):
     def test_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
+        self.assertEqual(md5(subtitles['en']), '26399116d23ae3cf2c087cea94bc43b4')
 
     def test_subtitles_lang(self):
         self.DL.params['writesubtitles'] = True

From 4e62ebe25013b07dd43e27eea37710b2c374455f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 20:56:32 +0100
Subject: [PATCH 022/974] [youtube] Try to extract the video_info from the
 webpage before requesting the 'get_video_info' pages

The YouTube player doesn't seem to use them except for embedded videos, so we can skip a network request.
But they still provide better error mesagges (for removed videos for example).
---
 youtube_dl/extractor/youtube.py | 60 +++++++++++++--------------------
 1 file changed, 24 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4b19061a9..6d48761c6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -675,7 +675,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             player_url = None
 
         # Get video info
-        self.report_video_info_webpage_download(video_id)
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
@@ -694,15 +693,30 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             video_info = compat_parse_qs(video_info_webpage)
         else:
             age_gate = False
-            for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                video_info_url = (proto + '://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                                  % (video_id, el_type))
-                video_info_webpage = self._download_webpage(video_info_url, video_id,
-                                                            note=False,
-                                                            errnote='unable to download video info webpage')
-                video_info = compat_parse_qs(video_info_webpage)
-                if 'token' in video_info:
-                    break
+            try:
+                # Try looking directly into the video webpage
+                mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
+                if not mobj:
+                    raise ValueError('Could not find ytplayer.config')  # caught below
+                json_code = uppercase_escape(mobj.group(1))
+                ytplayer_config = json.loads(json_code)
+                args = ytplayer_config['args']
+                # Convert to the same format returned by compat_parse_qs
+                video_info = dict((k, [v]) for k, v in args.items())
+                if 'url_encoded_fmt_stream_map' not in args:
+                    raise ValueError('No stream_map present')  # caught below
+            except ValueError:
+                # We fallback to the get_video_info pages (used by the embed page)
+                self.report_video_info_webpage_download(video_id)
+                for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
+                    video_info_url = (proto + '://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
+                        % (video_id, el_type))
+                    video_info_webpage = self._download_webpage(video_info_url,
+                        video_id, note=False,
+                        errnote='unable to download video info webpage')
+                    video_info = compat_parse_qs(video_info_webpage)
+                    if 'token' in video_info:
+                        break
         if 'token' not in video_info:
             if 'reason' in video_info:
                 raise ExtractorError(
@@ -827,32 +841,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         if self._downloader.params.get('writeannotations', False):
             video_annotations = self._extract_annotations(video_id)
 
-        # Decide which formats to download
-        try:
-            mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
-            if not mobj:
-                raise ValueError('Could not find vevo ID')
-            json_code = uppercase_escape(mobj.group(1))
-            ytplayer_config = json.loads(json_code)
-            args = ytplayer_config['args']
-            # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
-            # this signatures are encrypted
-            if 'url_encoded_fmt_stream_map' not in args:
-                raise ValueError('No stream_map present')  # caught below
-            re_signature = re.compile(r'[&,]s=')
-            m_s = re_signature.search(args['url_encoded_fmt_stream_map'])
-            if m_s is not None:
-                self.to_screen('%s: Encrypted signatures detected.' % video_id)
-                video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
-            m_s = re_signature.search(args.get('adaptive_fmts', ''))
-            if m_s is not None:
-                if 'adaptive_fmts' in video_info:
-                    video_info['adaptive_fmts'][0] += ',' + args['adaptive_fmts']
-                else:
-                    video_info['adaptive_fmts'] = [args['adaptive_fmts']]
-        except ValueError:
-            pass
-
         def _map_to_format_list(urlmap):
             formats = []
             for itag, video_real_url in urlmap.items():

From 7160532d41af171e6ffa41986b04bc51b934a5ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 21:07:50 +0100
Subject: [PATCH 023/974] [youtube] Simplify code for getting the dash manifest
 url

video_info contains now the 'ytplayer.config.args' dictionary
---
 youtube_dl/extractor/youtube.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6d48761c6..6268b012f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -933,10 +933,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 # However, in the case of an age restriction there won't be any embedded dashmpd in the video_webpage.
                 # Luckily, it seems, this case uses some kind of default signature (len == 86), so the
                 # combination of get_video_info and the _static_decrypt_signature() decryption fallback will work here.
-                if age_gate:
-                    dash_manifest_url = video_info.get('dashmpd')[0]
-                else:
-                    dash_manifest_url = ytplayer_config['args']['dashmpd']
+                dash_manifest_url = video_info.get('dashmpd')[0]
 
                 def decrypt_sig(mobj):
                     s = mobj.group(1)

From c522adb1f0747c5b7f4c8c60e6edcc6f1809f718 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 21:45:49 +0100
Subject: [PATCH 024/974] [youtube] Add a normal age-gate test video

---
 youtube_dl/extractor/youtube.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6268b012f..2642ecfff 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -402,7 +402,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'title': 'Burning Everyone\'s Koran',
                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
             }
-        }
+        },
+        # Normal age-gate video (No vevo, embed allowed)
+        {
+            'url': 'http://youtube.com/watch?v=HtVdAasjOgU',
+            'info_dict': {
+                'id': 'HtVdAasjOgU',
+                'ext': 'mp4',
+                'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
+                'description': 'md5:eca57043abae25130f58f655ad9a7771',
+                'uploader': 'The Witcher',
+                'uploader_id': 'WitcherGame',
+                'upload_date': '20140605',
+            },
+        },
     ]
 
     def __init__(self, *args, **kwargs):

From f2b44a2513d788e0e6a7f54497a7ac0e4090c1ff Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 19:06:23 +0100
Subject: [PATCH 025/974] [myspace] Use player_url for faster download It keeps
 reconnecting without it. Download time decreased from 7+ minutes to 25
 seconds for me.

---
 youtube_dl/extractor/myspace.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index c3c1a27c3..24da36212 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -48,6 +48,8 @@ class MySpaceIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         webpage = self._download_webpage(url, video_id)
+        player_url = self._search_regex(
+            r'playerSwf":"([^"?]*)', webpage, 'player URL')
 
         if mobj.group('mediatype').startswith('music/song'):
             # songs don't store any useful info in the 'context' variable
@@ -79,6 +81,7 @@ class MySpaceIE(InfoExtractor):
         info.update({
             'url': rtmp_url,
             'play_path': play_path,
+            'player_url': player_url,
             'ext': 'flv',
         })
         return info

From 03fd72d9967805aff7f28213385cf04e420d136e Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 19:07:36 +0100
Subject: [PATCH 026/974] [myspace] Add more data to info dict `uploader` is an
 artist `playlist` is an album

---
 youtube_dl/extractor/myspace.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 24da36212..1a118a37f 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -60,7 +60,9 @@ class MySpaceIE(InfoExtractor):
             info = {
                 'id': video_id,
                 'title': self._og_search_title(webpage),
+                'uploader': search_data('artist-name'),
                 'uploader_id': search_data('artist-username'),
+                'playlist': search_data('album-title'),
                 'thumbnail': self._og_search_thumbnail(webpage),
             }
         else:

From 1940fadd5307a11fd64ab26c43fbc31764c213a1 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 19:36:24 +0100
Subject: [PATCH 027/974] [myspace] Handle non-playable songs I'm adding this
 because sometimes there is a song page, but you cannot play it. Example:
 https://myspace.com/starset2/music/song/let-it-die-maniac-agenda-remix-bonus-track-95799916-106964439
 It will be useful for downloading whole album with songs like this.

---
 youtube_dl/extractor/myspace.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 1a118a37f..843e1f24a 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -53,9 +53,17 @@ class MySpaceIE(InfoExtractor):
 
         if mobj.group('mediatype').startswith('music/song'):
             # songs don't store any useful info in the 'context' variable
+            song_data = self._search_regex(
+                r'''<button.*data-song-id=(["\'])%s\1.*''' % video_id,
+                webpage, 'song_data', default=None, group=0)
+            if song_data is None:
+                self.to_screen(
+                    '%s: No downloadable song on this page' % video_id)
+                return
             def search_data(name):
                 return self._search_regex(
-                    r'data-%s="(.*?)"' % name, webpage, name)
+                    r'''data-%s=([\'"])(.*?)\1''' % name,
+                    song_data, name, default='', group=2)
             streamUrl = search_data('stream-url')
             info = {
                 'id': video_id,

From 3266f0c68e9512f226043d868630a7179f45d390 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 20:00:16 +0100
Subject: [PATCH 028/974] [myspace] Redirect to other extractors There are many
 songs just linked from Vevo/YouTube to MySpace. Vevo example:
 https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041
 YouTube example:
 https://myspace.com/starset2/music/song/first-light-95799905-106964426

---
 youtube_dl/extractor/myspace.py | 13 +++++++++++++
 youtube_dl/extractor/vevo.py    |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 843e1f24a..dd0a0415e 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
 )
+from ..utils import ExtractorError
 
 
 class MySpaceIE(InfoExtractor):
@@ -65,6 +66,18 @@ class MySpaceIE(InfoExtractor):
                     r'''data-%s=([\'"])(.*?)\1''' % name,
                     song_data, name, default='', group=2)
             streamUrl = search_data('stream-url')
+            if not streamUrl:
+                vevo_id = search_data('vevo-id')
+                youtube_id = search_data('youtube-id')
+                if vevo_id:
+                    self.to_screen('Vevo video detected: %s' % vevo_id)
+                    return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
+                elif youtube_id:
+                    self.to_screen('Youtube video detected: %s' % youtube_id)
+                    return self.url_result(youtube_id, ie='Youtube')
+                else:
+                    raise ExtractorError(
+                        'Found song but don\'t know how to download it')
             info = {
                 'id': video_id,
                 'title': self._og_search_title(webpage),
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 5b1a3ec78..c912c3cbe 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -13,7 +13,7 @@ from ..utils import (
 class VevoIE(InfoExtractor):
     """
     Accepts urls from vevo.com or in the format 'vevo:{id}'
-    (currently used by MTVIE)
+    (currently used by MTVIE and MySpaceIE)
     """
     _VALID_URL = r'''(?x)
         (?:https?://www\.vevo\.com/watch/(?:[^/]+/(?:[^/]+/)?)?|

From a196a53265a1df05270fe2a5ae83257b3629d30c Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 19:57:35 +0100
Subject: [PATCH 029/974] [myspace] Update tests

---
 youtube_dl/extractor/myspace.py | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index dd0a0415e..b4b1fb51d 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -29,19 +29,42 @@ class MySpaceIE(InfoExtractor):
                 'skip_download': True,
             },
         },
-        # song
+        # songs
         {
-            'url': 'https://myspace.com/spiderbags/music/song/darkness-in-my-heart-39008454-27041242',
+            'url': 'https://myspace.com/killsorrow/music/song/of-weakened-soul...-93388656-103880681',
+            'md5': 'f1d7323321f6b7775bf1e3754c1707dc',
             'info_dict': {
-                'id': '39008454',
+                'id': '93388656',
                 'ext': 'flv',
-                'title': 'Darkness In My Heart',
-                'uploader_id': 'spiderbags',
+                'playlist': 'The Demo',
+                'title': 'Of weakened soul...',
+                'uploader': 'Killsorrow',
+                'uploader_id': 'killsorrow',
             },
             'params': {
                 # rtmp download
                 'skip_download': True,
             },
+        }, {
+            'add_ie': ['Vevo'],
+            'url': 'https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041',
+            'info_dict': {
+                'id': u'USZM20600099',
+                'title': u'Animal I Have Become',
+                'uploader': u'Three Days Grace',
+                'timestamp': int,
+            },
+            'skip': 'VEVO is only available in some countries',
+        }, {
+            'add_ie': ['Youtube'],
+            'url': 'https://myspace.com/starset2/music/song/first-light-95799905-106964426',
+            'info_dict': {
+                'id': 'ypWvQgnJrSU',
+                'title': 'Starset - First Light',
+                'uploader': 'Jacob Soren',
+                'uploader_id': 'SorenPromotions',
+                'upload_date': '20140725',
+            }
         },
     ]
 

From 95c673a148f1a6cd8571b4dfa931d98092500fc2 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 21:36:00 +0100
Subject: [PATCH 030/974] [myspace] Add extractor for albums

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/myspace.py  | 43 ++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9387feef1..0339d1386 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -242,7 +242,7 @@ from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
 from .musicvault import MusicVaultIE
 from .muzu import MuzuTVIE
-from .myspace import MySpaceIE
+from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvideo import MyVideoIE
 from .naver import NaverIE
diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index b4b1fb51d..759bc7fed 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -131,3 +131,46 @@ class MySpaceIE(InfoExtractor):
             'ext': 'flv',
         })
         return info
+
+
+class MySpaceAlbumIE(InfoExtractor):
+    IE_NAME = 'MySpace:album'
+    _VALID_URL = r'https?://myspace\.com/([^/]+)/music/album/(?P<title>.*-)(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://myspace.com/starset2/music/album/transmissions-19455773',
+        'info_dict': {
+            'title': 'Transmissions',
+            'id': '19455773',
+        },
+        'playlist_count': 14,
+        'skip': 'this album is only available in some countries',
+    }, {
+        'url': 'https://myspace.com/killsorrow/music/album/the-demo-18596029',
+        'info_dict': {
+            'title': 'The Demo',
+            'id': '18596029',
+        },
+        'playlist_count': 5,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        playlist_id = mobj.group('id')
+        display_id = mobj.group('title') + playlist_id
+        webpage = self._download_webpage(url, display_id)
+        tracks_paths = re.findall(r'"music:song" content="(.*?)"', webpage)
+        if not tracks_paths:
+            self.to_screen('%s: No songs found, try using proxy' % display_id)
+            return
+        entries = [
+            self.url_result(t_path, ie=MySpaceIE.ie_key())
+            for t_path in tracks_paths]
+        title = self._og_search_title(webpage)
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'display_id': display_id,
+            'title': title,
+            'entries': entries,
+        }

From 19e92770c9fe5c08297f8100952d1bdb0e90db71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 23:44:37 +0100
Subject: [PATCH 031/974] [myspace] Replace removed test video and fix the
 others

---
 youtube_dl/extractor/myspace.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 759bc7fed..0ead81095 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -1,3 +1,4 @@
+# encoding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -15,14 +16,14 @@ class MySpaceIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'https://myspace.com/coldplay/video/viva-la-vida/100008689',
+            'url': 'https://myspace.com/fiveminutestothestage/video/little-big-town/109594919',
             'info_dict': {
-                'id': '100008689',
+                'id': '109594919',
                 'ext': 'flv',
-                'title': 'Viva La Vida',
-                'description': 'The official Viva La Vida video, directed by Hype Williams',
-                'uploader': 'Coldplay',
-                'uploader_id': 'coldplay',
+                'title': 'Little Big Town',
+                'description': 'This country quartet was all smiles while playing a sold out show at the Pacific Amphitheatre in Orange County, California.',
+                'uploader': 'Five Minutes to the Stage',
+                'uploader_id': 'fiveminutestothestage',
             },
             'params': {
                 # rtmp download
@@ -32,7 +33,6 @@ class MySpaceIE(InfoExtractor):
         # songs
         {
             'url': 'https://myspace.com/killsorrow/music/song/of-weakened-soul...-93388656-103880681',
-            'md5': 'f1d7323321f6b7775bf1e3754c1707dc',
             'info_dict': {
                 'id': '93388656',
                 'ext': 'flv',
@@ -49,10 +49,12 @@ class MySpaceIE(InfoExtractor):
             'add_ie': ['Vevo'],
             'url': 'https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041',
             'info_dict': {
-                'id': u'USZM20600099',
-                'title': u'Animal I Have Become',
-                'uploader': u'Three Days Grace',
+                'id': 'USZM20600099',
+                'ext': 'mp4',
+                'title': 'Animal I Have Become',
+                'uploader': 'Three Days Grace',
                 'timestamp': int,
+                'upload_date': '20060502',
             },
             'skip': 'VEVO is only available in some countries',
         }, {
@@ -60,7 +62,9 @@ class MySpaceIE(InfoExtractor):
             'url': 'https://myspace.com/starset2/music/song/first-light-95799905-106964426',
             'info_dict': {
                 'id': 'ypWvQgnJrSU',
+                'ext': 'mp4',
                 'title': 'Starset - First Light',
+                'description': 'md5:2d5db6c9d11d527683bcda818d332414',
                 'uploader': 'Jacob Soren',
                 'uploader_id': 'SorenPromotions',
                 'upload_date': '20140725',

From 954f36f890713f39bba7e5a08559856c367c89e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 1 Dec 2014 00:03:24 +0100
Subject: [PATCH 032/974] [myspace] Cleanup

---
 youtube_dl/extractor/myspace.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 0ead81095..e62614670 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -36,7 +36,6 @@ class MySpaceIE(InfoExtractor):
             'info_dict': {
                 'id': '93388656',
                 'ext': 'flv',
-                'playlist': 'The Demo',
                 'title': 'Of weakened soul...',
                 'uploader': 'Killsorrow',
                 'uploader_id': 'killsorrow',
@@ -85,13 +84,14 @@ class MySpaceIE(InfoExtractor):
                 r'''<button.*data-song-id=(["\'])%s\1.*''' % video_id,
                 webpage, 'song_data', default=None, group=0)
             if song_data is None:
-                self.to_screen(
+                # some songs in an album are not playable
+                self.report_warning(
                     '%s: No downloadable song on this page' % video_id)
                 return
             def search_data(name):
                 return self._search_regex(
-                    r'''data-%s=([\'"])(.*?)\1''' % name,
-                    song_data, name, default='', group=2)
+                    r'''data-%s=([\'"])(?P<data>.*?)\1''' % name,
+                    song_data, name, default='', group='data')
             streamUrl = search_data('stream-url')
             if not streamUrl:
                 vevo_id = search_data('vevo-id')
@@ -110,7 +110,6 @@ class MySpaceIE(InfoExtractor):
                 'title': self._og_search_title(webpage),
                 'uploader': search_data('artist-name'),
                 'uploader_id': search_data('artist-username'),
-                'playlist': search_data('album-title'),
                 'thumbnail': self._og_search_thumbnail(webpage),
             }
         else:
@@ -165,16 +164,16 @@ class MySpaceAlbumIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
         tracks_paths = re.findall(r'"music:song" content="(.*?)"', webpage)
         if not tracks_paths:
-            self.to_screen('%s: No songs found, try using proxy' % display_id)
-            return
+            raise ExtractorError(
+                '%s: No songs found, try using proxy' % display_id,
+                expected=True)
         entries = [
             self.url_result(t_path, ie=MySpaceIE.ie_key())
             for t_path in tracks_paths]
-        title = self._og_search_title(webpage)
         return {
             '_type': 'playlist',
             'id': playlist_id,
             'display_id': display_id,
-            'title': title,
+            'title': self._og_search_title(webpage),
             'entries': entries,
         }

From d128cfe3931eeb779a13678e43e09e012a64aca7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Dec 2014 20:18:42 +0600
Subject: [PATCH 033/974] [slideshare] Fix description extraction

---
 youtube_dl/extractor/slideshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index 53c3c9220..5864b9936 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -39,7 +39,7 @@ class SlideshareIE(InfoExtractor):
         ext = info['jsplayer']['video_extension']
         video_url = compat_urlparse.urljoin(bucket, doc + '-SD.' + ext)
         description = self._html_search_regex(
-            r'<p\s+(?:style="[^"]*"\s+)?class="description.*?"[^>]*>(.*?)</p>', webpage,
+            r'<p\s+(?:style="[^"]*"\s+)?class=".*?description.*?"[^>]*>(.*?)</p>', webpage,
             'description', fatal=False)
 
         return {

From 94aa25b99541be80a975fd48c07d096fc8c3203c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 1 Dec 2014 16:15:09 +0100
Subject: [PATCH 034/974] Credit @Tithen-Firion for the myspace changes (#4341)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d4b0b75fa..bd2e967e3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -88,3 +88,4 @@ Dao Hoang Son
 Oskar Jauch
 Matthew Rayfield
 t0mm0
+Tithen-Firion

From 1434bffa1ff2fb1e4d80efc900a13a3541923119 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 1 Dec 2014 18:10:04 +0200
Subject: [PATCH 035/974] [tunein] Use station API

---
 youtube_dl/extractor/tunein.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/tunein.py b/youtube_dl/extractor/tunein.py
index 8c29f1634..4ce5aeeba 100644
--- a/youtube_dl/extractor/tunein.py
+++ b/youtube_dl/extractor/tunein.py
@@ -19,6 +19,7 @@ class TuneInIE(InfoExtractor):
         |tun\.in/(?P<redirect_id>[A-Za-z0-9]+)
     )
     '''
+    _API_URL_TEMPLATE = 'http://tunein.com/tuner/tune/?stationId={0:}&tuneType=Station'
 
     _INFO_DICT = {
         'id': '34682',
@@ -56,13 +57,10 @@ class TuneInIE(InfoExtractor):
             mobj = re.match(self._VALID_URL, url)
         station_id = mobj.group('id')
 
-        webpage = self._download_webpage(
-            url, station_id, note='Downloading station webpage')
+        station_info = self._download_json(
+            self._API_URL_TEMPLATE.format(station_id),
+            station_id, note='Downloading station JSON')
 
-        payload = self._html_search_regex(
-            r'(?m)TuneIn\.payload\s*=\s*(\{[^$]+?)$', webpage, 'JSON data')
-        json_data = json.loads(payload)
-        station_info = json_data['Station']['broadcast']
         title = station_info['Title']
         thumbnail = station_info.get('Logo')
         location = station_info.get('Location')

From df015c69eab2e7617664ed2d2982832e08c8b843 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 1 Dec 2014 17:28:34 +0100
Subject: [PATCH 036/974] release 2014.12.01

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9476bedfa..847a47c45 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.11.27'
+__version__ = '2014.12.01'

From c41a9650c3b8c5430cd21661d668ac9fb346e397 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 1 Dec 2014 17:35:03 +0100
Subject: [PATCH 037/974] [youtube] Extract framerate from the dash manifest

Not all videos have 60 fps, for example they can have 48 fps.
---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2642ecfff..1cba40387 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -971,6 +971,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                         'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
                         'asr': int_or_none(r.attrib.get('audioSamplingRate')),
                         'filesize': filesize,
+                        'fps': int_or_none(r.attrib.get('frameRate')),
                     }
                     try:
                         existing_format = next(

From 05177b34a62e3a45417ec1937650d43c20ebdfab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 1 Dec 2014 18:20:36 +0100
Subject: [PATCH 038/974] [rutube] Extract m3u8 formats (fixes #3984)

---
 youtube_dl/extractor/rutube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 0c8790da2..6941d96fb 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -53,6 +53,7 @@ class RutubeIE(InfoExtractor):
         m3u8_url = options['video_balancer'].get('m3u8')
         if m3u8_url is None:
             raise ExtractorError('Couldn\'t find m3u8 manifest url')
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
 
         return {
             'id': video['id'],
@@ -60,8 +61,7 @@ class RutubeIE(InfoExtractor):
             'description': video['description'],
             'duration': video['duration'],
             'view_count': video['hits'],
-            'url': m3u8_url,
-            'ext': 'mp4',
+            'formats': formats,
             'thumbnail': video['thumbnail_url'],
             'uploader': author.get('name'),
             'uploader_id': compat_str(author['id']) if author else None,

From b1399a144deb810a6d599e500391befe93142cf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Dec 2014 20:45:43 +0600
Subject: [PATCH 039/974] [rts] Add support for the new URL format and extract
 display id (Closes #4349)

---
 youtube_dl/extractor/rts.py | 44 ++++++++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index e8199b114..82d783078 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class RTSIE(InfoExtractor):
     IE_DESC = 'RTS.ch'
-    _VALID_URL = r'^https?://(?:www\.)?rts\.ch/(?:[^/]+/){2,}(?P<id>[0-9]+)-.*?\.html'
+    _VALID_URL = r'^https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/-/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
 
     _TESTS = [
         {
@@ -23,6 +23,7 @@ class RTSIE(InfoExtractor):
             'md5': '753b877968ad8afaeddccc374d4256a5',
             'info_dict': {
                 'id': '3449373',
+                'display_id': 'les-enfants-terribles',
                 'ext': 'mp4',
                 'duration': 1488,
                 'title': 'Les Enfants Terribles',
@@ -30,7 +31,8 @@ class RTSIE(InfoExtractor):
                 'uploader': 'Divers',
                 'upload_date': '19680921',
                 'timestamp': -40280400,
-                'thumbnail': 're:^https?://.*\.image'
+                'thumbnail': 're:^https?://.*\.image',
+                'view_count': int,
             },
         },
         {
@@ -38,6 +40,7 @@ class RTSIE(InfoExtractor):
             'md5': 'c148457a27bdc9e5b1ffe081a7a8337b',
             'info_dict': {
                 'id': '5624067',
+                'display_id': 'entre-ciel-et-mer',
                 'ext': 'mp4',
                 'duration': 3720,
                 'title': 'Les yeux dans les cieux - Mon homard au Canada',
@@ -45,7 +48,8 @@ class RTSIE(InfoExtractor):
                 'uploader': 'Passe-moi les jumelles',
                 'upload_date': '20140404',
                 'timestamp': 1396635300,
-                'thumbnail': 're:^https?://.*\.image'
+                'thumbnail': 're:^https?://.*\.image',
+                'view_count': int,
             },
         },
         {
@@ -53,6 +57,7 @@ class RTSIE(InfoExtractor):
             'md5': 'b4326fecd3eb64a458ba73c73e91299d',
             'info_dict': {
                 'id': '5745975',
+                'display_id': '1-2-kloten-fribourg-5-2-second-but-pour-gotteron-par-kwiatowski',
                 'ext': 'mp4',
                 'duration': 48,
                 'title': '1/2, Kloten - Fribourg (5-2): second but pour Gottéron par Kwiatowski',
@@ -60,7 +65,8 @@ class RTSIE(InfoExtractor):
                 'uploader': 'Hockey',
                 'upload_date': '20140403',
                 'timestamp': 1396556882,
-                'thumbnail': 're:^https?://.*\.image'
+                'thumbnail': 're:^https?://.*\.image',
+                'view_count': int,
             },
             'skip': 'Blocked outside Switzerland',
         },
@@ -69,6 +75,7 @@ class RTSIE(InfoExtractor):
             'md5': '9bb06503773c07ce83d3cbd793cebb91',
             'info_dict': {
                 'id': '5745356',
+                'display_id': 'londres-cachee-par-un-epais-smog',
                 'ext': 'mp4',
                 'duration': 33,
                 'title': 'Londres cachée par un épais smog',
@@ -76,7 +83,8 @@ class RTSIE(InfoExtractor):
                 'uploader': 'Le Journal en continu',
                 'upload_date': '20140403',
                 'timestamp': 1396537322,
-                'thumbnail': 're:^https?://.*\.image'
+                'thumbnail': 're:^https?://.*\.image',
+                'view_count': int,
             },
         },
         {
@@ -84,6 +92,7 @@ class RTSIE(InfoExtractor):
             'md5': 'dd8ef6a22dff163d063e2a52bc8adcae',
             'info_dict': {
                 'id': '5706148',
+                'display_id': 'urban-hippie-de-damien-krisl-03-04-2014',
                 'ext': 'mp3',
                 'duration': 123,
                 'title': '"Urban Hippie", de Damien Krisl',
@@ -92,22 +101,40 @@ class RTSIE(InfoExtractor):
                 'timestamp': 1396551600,
             },
         },
+        {
+            'url': 'http://www.rts.ch/play/tv/-/video/le-19h30?id=6348260',
+            'md5': '968777c8779e5aa2434be96c54e19743',
+            'info_dict': {
+                'id': '6348260',
+                'display_id': 'le-19h30',
+                'ext': 'mp4',
+                'duration': 1796,
+                'title': 'Le 19h30',
+                'description': '',
+                'uploader': 'Le 19h30',
+                'upload_date': '20141201',
+                'timestamp': 1417458600,
+                'thumbnail': 're:^https?://.*\.image',
+                'view_count': int,
+            },
+        }
     ]
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
+        video_id = m.group('id') or m.group('id_new')
+        display_id = m.group('display_id') or m.group('display_id_new')
 
         def download_json(internal_id):
             return self._download_json(
                 'http://www.rts.ch/a/%s.html?f=json/article' % internal_id,
-                video_id)
+                display_id)
 
         all_info = download_json(video_id)
 
         # video_id extracted out of URL is not always a real id
         if 'video' not in all_info and 'audio' not in all_info:
-            page = self._download_webpage(url, video_id)
+            page = self._download_webpage(url, display_id)
             internal_id = self._html_search_regex(
                 r'<(?:video|audio) data-id="([0-9]+)"', page,
                 'internal video id')
@@ -143,6 +170,7 @@ class RTSIE(InfoExtractor):
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'formats': formats,
             'title': info['title'],
             'description': info.get('intro'),

From 752c8c9b7635137c519ac9b0872c7155322e51f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Dec 2014 20:53:19 +0600
Subject: [PATCH 040/974] [rts] Improve _VALID_URL

---
 youtube_dl/extractor/rts.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index 82d783078..c31049c0e 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class RTSIE(InfoExtractor):
     IE_DESC = 'RTS.ch'
-    _VALID_URL = r'^https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/-/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
+    _VALID_URL = r'^https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
 
     _TESTS = [
         {
@@ -117,6 +117,10 @@ class RTSIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.image',
                 'view_count': int,
             },
+        },
+        {
+            'url': 'http://www.rts.ch/play/tv/le-19h30/video/le-chantier-du-nouveau-parlement-vaudois-a-permis-une-trouvaille-historique?id=6348280',
+            'only_matching': True,
         }
     ]
 

From 706d7d4ee737f0e382df0bae3c8146dd5839995d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Dec 2014 21:18:07 +0600
Subject: [PATCH 041/974] [YoutubeDL] Avoid negative timestamps on Windows

---
 youtube_dl/YoutubeDL.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 21c7c298a..f89ac4e1d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -787,6 +787,10 @@ class YoutubeDL(object):
             info_dict['display_id'] = info_dict['id']
 
         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
+            # Working around negative timestamps in Windows
+            # (see http://bugs.python.org/issue1646728)
+            if info_dict['timestamp'] < 0 and os.name == 'nt':
+                info_dict['timestamp'] = 0
             upload_date = datetime.datetime.utcfromtimestamp(
                 info_dict['timestamp'])
             info_dict['upload_date'] = upload_date.strftime('%Y%m%d')

From ac5633592afa9d70c05170333a0fb9013ba868da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Dec 2014 22:23:23 +0600
Subject: [PATCH 042/974] [24video] Add extractor (Closes #4350)

---
 youtube_dl/extractor/__init__.py        |   1 +
 youtube_dl/extractor/twentyfourvideo.py | 109 ++++++++++++++++++++++++
 2 files changed, 110 insertions(+)
 create mode 100644 youtube_dl/extractor/twentyfourvideo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0339d1386..8b513ffd1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -417,6 +417,7 @@ from .tutv import TutvIE
 from .tvigle import TvigleIE
 from .tvp import TvpIE
 from .tvplay import TVPlayIE
+from .twentyfourvideo import TwentyFourVideoIE
 from .twitch import TwitchIE
 from .ubu import UbuIE
 from .udemy import (
diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
new file mode 100644
index 000000000..67e8bfea0
--- /dev/null
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -0,0 +1,109 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    int_or_none,
+)
+
+
+class TwentyFourVideoIE(InfoExtractor):
+    IE_NAME = '24video'
+    _VALID_URL = r'https?://(?:www\.)?24video\.net/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
+
+    _TESTS = [
+        {
+            'url': 'http://www.24video.net/video/view/1044982',
+            'md5': '48dd7646775690a80447a8dca6a2df76',
+            'info_dict': {
+                'id': '1044982',
+                'ext': 'mp4',
+                'title': 'Эротика каменного века',
+                'description': 'Как смотрели порно в каменном веке.',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'uploader': 'SUPERTELO',
+                'duration': 31,
+                'timestamp': 1275937857,
+                'upload_date': '20100607',
+                'age_limit': 18,
+                'like_count': int,
+                'dislike_count': int,
+            },
+        },
+        {
+            'url': 'http://www.24video.net/player/new24_play.swf?id=1044982',
+            'only_matching': True,
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://www.24video.net/video/view/%s' % video_id, video_id)
+
+        title = self._og_search_title(webpage)
+        description = self._html_search_regex(
+            r'<span itemprop="description">([^<]+)</span>', webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
+        duration = int_or_none(self._og_search_property(
+            'duration', webpage, 'duration', fatal=False))
+        timestamp = parse_iso8601(self._search_regex(
+            r'<time id="video-timeago" datetime="([^"]+)" itemprop="uploadDate">',
+            webpage, 'upload date'))
+
+        uploader = self._html_search_regex(
+            r'Загрузил\s*<a href="/jsecUser/movies/[^"]+" class="link">([^<]+)</a>',
+            webpage, 'uploader', fatal=False)
+
+        view_count = int_or_none(self._html_search_regex(
+            r'<span class="video-views">(\d+) просмотр',
+            webpage, 'view count', fatal=False))
+        comment_count = int_or_none(self._html_search_regex(
+            r'<div class="comments-title" id="comments-count">(\d+) комментари',
+            webpage, 'comment count', fatal=False))
+
+        formats = []
+
+        pc_video = self._download_xml(
+            'http://www.24video.net/video/xml/%s?mode=play' % video_id,
+            video_id, 'Downloading PC video URL').find('.//video')
+
+        formats.append({
+            'url': pc_video.attrib['url'],
+            'format_id': 'pc',
+            'quality': 1,
+        })
+
+        like_count = int_or_none(pc_video.get('ratingPlus'))
+        dislike_count = int_or_none(pc_video.get('ratingMinus'))
+        age_limit = 18 if pc_video.get('adult') == 'true' else 0
+
+        mobile_video = self._download_xml(
+            'http://www.24video.net/video/xml/%s' % video_id,
+            video_id, 'Downloading mobile video URL').find('.//video')
+
+        formats.append({
+            'url': mobile_video.attrib['url'],
+            'format_id': 'mobile',
+            'quality': 0,
+        })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'duration': duration,
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'like_count': like_count,
+            'dislike_count': dislike_count,
+            'age_limit': age_limit,
+            'formats': formats,
+        }

From c492970b4b2d9b3afa5fe817e4da748816126ebd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Dec 2014 22:24:47 +0600
Subject: [PATCH 043/974] [rts] Improve _VALID_URL

---
 youtube_dl/extractor/rts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index c31049c0e..dc59a5e5c 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class RTSIE(InfoExtractor):
     IE_DESC = 'RTS.ch'
-    _VALID_URL = r'^https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
+    _VALID_URL = r'https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
 
     _TESTS = [
         {

From 62d8b56655eceb763fe0150866092ef99ec6fb84 Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Tue, 2 Dec 2014 23:23:18 +0100
Subject: [PATCH 044/974] [nhl] Support videos which don't have mp4-extension
 (fixes #4348)

---
 youtube_dl/extractor/nhl.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index bdcf7e239..72dcbf424 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
     compat_urllib_parse,
+    compat_urllib_parse_urlparse
 )
 from ..utils import (
     unified_strdate,
@@ -24,9 +25,13 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
         initial_video_url = info['publishPoint']
         if info['formats'] == '1':
+            parsed_url = compat_urllib_parse_urlparse(initial_video_url)
+            path = parsed_url.path
+            extension_index = path.rfind('.')
+            path = path[:extension_index] + '_sd' + path[extension_index:]
             data = compat_urllib_parse.urlencode({
                 'type': 'fvod',
-                'path': initial_video_url.replace('.mp4', '_sd.mp4'),
+                'path': compat_urlparse.urlunparse(parsed_url[:2] + (path,) + parsed_url[3:])
             })
             path_url = 'http://video.nhl.com/videocenter/servlets/encryptvideopath?' + data
             path_doc = self._download_xml(
@@ -73,6 +78,17 @@ class NHLIE(NHLBaseInfoExtractor):
             'duration': 0,
             'upload_date': '20141011',
         },
+    }, {
+        'url': 'http://video.mapleleafs.nhl.com/videocenter/console?id=58665&catid=802',
+        'md5': 'c78fc64ea01777e426cfc202b746c825',
+        'info_dict': {
+            'id': '58665',
+            'ext': 'flv',
+            'title': 'Classic Game In Six - April 22, 1979',
+            'description': 'It was the last playoff game for the Leafs in the decade, and the last time the Leafs and Habs played in the playoffs. Great game, not a great ending.',
+            'duration': 400,
+            'upload_date': '20100129'
+        },
     }, {
         'url': 'http://video.flames.nhl.com/videocenter/console?id=630616',
         'only_matching': True,

From ea2ee40357eb2c43f8968a4eeba47e1cf97667ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 2 Dec 2014 23:56:30 +0100
Subject: [PATCH 045/974] [nhl.com:videocenter] Don't match url with 'id=*'
 before 'catid' in the query

Since the order extractors are added is not defined, it would match instead of NHLIE.
---
 youtube_dl/extractor/nhl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index bdcf7e239..2a84f6761 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -90,7 +90,7 @@ class NHLIE(NHLBaseInfoExtractor):
 class NHLVideocenterIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com:videocenter'
     IE_DESC = 'NHL videocenter category'
-    _VALID_URL = r'https?://video\.(?P<team>[^.]*)\.nhl\.com/videocenter/(console\?.*?catid=(?P<catid>[0-9]+)(?![&?]id=).*?)?$'
+    _VALID_URL = r'https?://video\.(?P<team>[^.]*)\.nhl\.com/videocenter/(console\?[^(id=)]*catid=(?P<catid>[0-9]+)(?![&?]id=).*?)?$'
     _TEST = {
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=999',
         'info_dict': {

From 163c8babaab947d243f4b8cc4e1fa0bc4b4e1603 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 3 Dec 2014 00:08:26 +0100
Subject: [PATCH 046/974] [nhl] Simplify

---
 youtube_dl/extractor/nhl.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 7d74b110b..0244368e9 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -26,9 +26,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
         initial_video_url = info['publishPoint']
         if info['formats'] == '1':
             parsed_url = compat_urllib_parse_urlparse(initial_video_url)
-            path = parsed_url.path
-            extension_index = path.rfind('.')
-            path = path[:extension_index] + '_sd' + path[extension_index:]
+            path = parsed_url.path.replace('.', '_sd.', 1)
             data = compat_urllib_parse.urlencode({
                 'type': 'fvod',
                 'path': compat_urlparse.urlunparse(parsed_url[:2] + (path,) + parsed_url[3:])

From c627f7d48c8e778958aaf03eb091364fbe12f2ca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 3 Dec 2014 12:15:34 +0100
Subject: [PATCH 047/974] release 2014.12.03

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 847a47c45..61902a8cc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.01'
+__version__ = '2014.12.03'

From 233d37fb6b420b01315c48488c84dc5b6e7aaea0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 3 Dec 2014 13:25:49 +0100
Subject: [PATCH 048/974] [brightcove] Make sure that the 'ext' variable is set
 (fixes #4360)

---
 youtube_dl/extractor/brightcove.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 2db7f9fef..bf18a97e0 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -265,6 +265,7 @@ class BrightcoveIE(InfoExtractor):
                 url = rend['defaultURL']
                 if not url:
                     continue
+                ext = None
                 if rend['remote']:
                     url_comp = compat_urllib_parse_urlparse(url)
                     if url_comp.path.endswith('.m3u8'):
@@ -276,7 +277,7 @@ class BrightcoveIE(InfoExtractor):
                         # akamaihd.net, but they don't use f4m manifests
                         url = url.replace('control/', '') + '?&v=3.3.0&fp=13&r=FEEFJ&g=RTSJIMBMPFPB'
                         ext = 'flv'
-                else:
+                if ext is None:
                     ext = determine_ext(url)
                 size = rend.get('size')
                 formats.append({

From de9bd74bc2e06a6d5396151050fd1eeafdd54a33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Dec 2014 21:17:11 +0600
Subject: [PATCH 049/974] [ted] Fix type_watch links extraction

---
 youtube_dl/extractor/ted.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index f8a87afda..72160503c 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -199,8 +199,9 @@ class TEDIE(SubtitlesInfoExtractor):
         webpage = self._download_webpage(url, name)
 
         config_json = self._html_search_regex(
-            r"data-config='([^']+)", webpage, 'config')
-        config = json.loads(config_json)
+            r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
+            webpage, 'config')
+        config = json.loads(config_json)['config']
         video_url = config['video']['url']
         thumbnail = config.get('image', {}).get('url')
 

From 0403b06985bcb004efb3683bca6a56a47e3227b1 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Wed, 3 Dec 2014 11:49:53 +0100
Subject: [PATCH 050/974] [soundcloud] Improve_VALID_URL Add support for links
 from Audiomack

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 3c1d058db..ab9483d2d 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -32,7 +32,7 @@ class SoundcloudIE(InfoExtractor):
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
-                          (?:/?\?secret_token=(?P<secret_token>[^&]+?))?$)
+                          (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
                        |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
                     )
                     '''

From 9095aa38ac7361b55aa0e9fd3fe4fccf23718e6c Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Wed, 3 Dec 2014 14:53:12 +0100
Subject: [PATCH 051/974] [audiomack] Update test

---
 youtube_dl/extractor/audiomack.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index eeeec768f..04386f7f7 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -24,17 +24,17 @@ class AudiomackIE(InfoExtractor):
         },
         # hosted on soundcloud via audiomack
         {
+            'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
             'file': '172419696.mp3',
             'info_dict':
             {
-                'ext': 'mp3',
+                'description': 'md5:1fc3272ed7a635cce5be1568c2822997',
                 'title': 'Young Thug ft Lil Wayne - Take Kare',
-                "upload_date": "20141016",
-                "description": "New track produced by London On Da Track called “Take Kare\"\n\nhttp://instagram.com/theyoungthugworld\nhttps://www.facebook.com/ThuggerThuggerCashMoney\n",
-                "uploader": "Young Thug World"
+                'uploader':'Young Thug World',
+                'upload_date':'20141016',
             }
-        }
+        },
     ]
 
     def _real_extract(self, url):

From dce2a3cf9e4640ec9ae54f1997596d00591a97c3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 01:33:30 +0100
Subject: [PATCH 052/974] [break] Remove md5sum from test

---
 youtube_dl/extractor/breakcom.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py
index 2c0e5eea2..4bcc897c9 100644
--- a/youtube_dl/extractor/breakcom.py
+++ b/youtube_dl/extractor/breakcom.py
@@ -14,7 +14,6 @@ class BreakIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?break\.com/video/(?:[^/]+/)*.+-(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056',
-        'md5': '33aa4ff477ecd124d18d7b5d23b87ce5',
         'info_dict': {
             'id': '2468056',
             'ext': 'mp4',

From eecd6a467dd6b28cd0cdd63a9cebd5a8bfafd4c4 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Wed, 3 Dec 2014 19:07:09 +0100
Subject: [PATCH 053/974] [vgtv] Update tests

---
 youtube_dl/extractor/vgtv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 70578a4cc..2f111bf7e 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -17,7 +17,7 @@ class VGTVIE(InfoExtractor):
             'info_dict': {
                 'id': '84196',
                 'ext': 'mp4',
-                'title': 'Hevnen er søt episode 1:10 - Abu',
+                'title': 'Hevnen er søt: Episode 10 - Abu',
                 'description': 'md5:e25e4badb5f544b04341e14abdc72234',
                 'thumbnail': 're:^https?://.*\.jpg',
                 'duration': 648.000,
@@ -35,7 +35,7 @@ class VGTVIE(InfoExtractor):
                 'title': 'OPPTAK: VGTV følger EM-kvalifiseringen',
                 'description': 'md5:3772d9c0dc2dff92a886b60039a7d4d3',
                 'thumbnail': 're:^https?://.*\.jpg',
-                'duration': 9056.000,
+                'duration': 9103.0,
                 'timestamp': 1410113864,
                 'upload_date': '20140907',
                 'view_count': int,

From 29f400b97d00cc7bd8b1a7549417584a45e38df8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 4 Dec 2014 02:54:25 +0100
Subject: [PATCH 054/974] [tvp] Update extractor

---
 youtube_dl/extractor/tvp.py | 121 +++++++++++++++++++++++++++++-------
 1 file changed, 98 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index a64580005..6b95e2ed1 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -1,37 +1,112 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class TvpIE(InfoExtractor):
     IE_NAME = 'tvp.pl'
-    _VALID_URL = r'https?://www\.tvp\.pl/.*?wideo/(?P<date>\d+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?P<type>vod|www)\.tvp\.pl/.*/(?P<id>\d+)$'
 
-    _TEST = {
-        'url': 'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
-        'md5': '148408967a6a468953c0a75cbdaf0d7a',
-        'info_dict': {
-            'id': '12878238',
-            'ext': 'wmv',
-            'title': '31.10.2013 - Odcinek 2',
-            'description': '31.10.2013 - Odcinek 2',
+    _TESTS = [
+        {
+            'url': 'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
+            'info_dict': {
+                'id': '12878238',
+                'ext': 'wmv',
+                'title': 'CAMPUSnews, 31.10.2013 - Odcinek 2',
+                'description': '',
+            },
+            'skip': 'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.',
+        }, {
+            'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem/wideo/odc-2/4278035',
+            'info_dict': {
+                'id': '4278035',
+                'ext': 'wmv',
+                'title': 'Ogniem i mieczem, odc. 2',
+                'description': 'Bohun dowiaduje się o złamaniu przez kniahinię danego mu słowa i wyrusza do Rozłogów. Helenie w ostatniej chwili udaje się uciec dzięki pomocy Zagłoby.',
+            },
+            'skip': 'As above',
+        }, {
+            'url': 'http://vod.tvp.pl/seriale/obyczajowe/czas-honoru/sezon-1-1-13/i-seria-odc-13/194536',
+            'info_dict': {
+                'id': '194536',
+                'ext': 'mp4',
+                'title': 'Czas honoru, I seria – odc. 13',
+                'description': 'WŁADEK\nCzesław prosi Marię o dostarczenie Władkowi zarazki tyfusu. Jeśli zachoruje zostanie przewieziony do szpitala skąd łatwiej będzie go odbić. Czy matka zdecyduje się zarazić syna? Karol odwiedza Wandę przyznaje się, że ją oszukiwał, ale ostrzega też, że grozi jej aresztowanie i nalega, żeby wyjechała z Warszawy. Czy dziewczyna zdecyduje się znów oddalić od ukochanego? Rozpoczyna się akcja odbicia Władka.',
+            },
+        }, {
+            'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
+            'info_dict': {
+                'id': '17916176',
+                'ext': 'mp4',
+                'title': 'rozmaitosci, TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
+                'description': '',
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': 'true',
+            },
+        }, {
+            'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
+            'info_dict': {
+                'id': '17834272',
+                'ext': 'mp4',
+                'title': 'Na sygnale, odc. 39',
+                'description': 'Ekipa Wiktora ratuje młodą matkę, która spadła ze schodów trzymając na rękach noworodka. Okazuje się, że dziewczyna jest surogatką, a biologiczni rodzice dziecka próbują zmusić ją do oddania synka…',
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': 'true',
+            },
         },
-        'skip': 'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.'
-    }
+    ]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        json_url = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id
-        params = self._download_json(
-            json_url, video_id, "Downloading video metadata")
-        video_url = params['video_url']
-
-        return {
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        webpage = self._download_webpage(
+            'http://www.tvp.pl/sess/tvplayer.php?object_id=%s' % video_id, video_id)
+        title = self._og_search_title(webpage)
+        series = self._search_regex(
+            r'{name:\s*([\'"])SeriesTitle\1,\s*value:\s*\1(?P<series>.*?)\1},',
+            webpage, 'series', group='series', default=None)
+        if series is not None and series not in title:
+            title = '%s, %s' % (series, title)
+        info_dict = {
             'id': video_id,
-            'title': self._og_search_title(webpage),
-            'ext': 'wmv',
-            'url': video_url,
-            'description': self._og_search_description(webpage),
+            'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
+            'description': self._og_search_description(webpage, default=''),
         }
+        if mobj.group('type') == 'vod' and info_dict['description'] == '':
+            info_dict.update({
+                'description': self._html_search_regex(
+                    r'(?s)<div\s+class=[\'"]opis.*?</div>',
+                    self._download_webpage(url, video_id), 'description', group=0),
+            })
+
+        video_url = self._search_regex(
+            r'0:{src:([\'"])(?P<url>.*?)\1', webpage, 'formats', group='url', default=None)
+        if video_url is None:
+            video_url = self._download_json(
+                'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id,
+                video_id)['video_url']
+
+        ext = video_url.rsplit('.', 1)[-1]
+        if ext != 'ism/manifest':
+            if '/' in ext:
+                ext = 'mp4'
+            info_dict.update({
+                'ext': ext,
+                'url': video_url,
+            })
+        else:
+            m3u8_url = re.sub('([^/]*)\.ism/manifest', r'\1.ism/\1.m3u8', video_url)
+            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+            info_dict.update({
+                'formats': formats,
+            })
+        return info_dict

From 6ce2c6783b2c1516fe9b1b5cb88d28231db45f8c Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 4 Dec 2014 05:14:09 +0100
Subject: [PATCH 055/974] [tvp] Add extractor

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/tvp.py      | 56 ++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8b513ffd1..b09ee303d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -415,7 +415,7 @@ from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
 from .tvigle import TvigleIE
-from .tvp import TvpIE
+from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twitch import TwitchIE
diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 6b95e2ed1..2248a9fdf 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -110,3 +110,59 @@ class TvpIE(InfoExtractor):
                 'formats': formats,
             })
         return info_dict
+
+
+class TvpSeriesIE(InfoExtractor):
+    IE_NAME = 'tvp.pl:Series'
+    _VALID_URL = r'https?://vod\.tvp\.pl/(?:[^/]+/){2}(?P<id>[^/]+)/?$'
+
+    _TESTS = [
+        {
+            'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem',
+            'info_dict': {
+                'title': 'Ogniem i mieczem',
+                'id': '4278026',
+            },
+            'playlist_count': 4,
+        }, {
+            'url': 'http://vod.tvp.pl/audycje/podroze/boso-przez-swiat',
+            'info_dict': {
+                'title': 'Boso przez świat',
+                'id': '9329207',
+            },
+            'playlist_count': 86,
+        }
+    ]
+
+    def _force_download_webpage(self, url, v_id, tries=0):
+        if tries >= 5:
+            raise ExtractorError(
+                '%s: Cannot download webpage, try again later' % v_id)
+        # Sometimes happen, but in my tests second try always succeeded
+        try:
+            return self._download_webpage(url, v_id)
+        except IncompleteRead as e:
+            return self._force_download_webpage(url, v_id, tries+1)
+    
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._force_download_webpage(url, display_id)
+        title = self._html_search_regex(
+            r'(?s) id=[\'"]path[\'"]>(.*?)</span>', webpage, 'series')
+        title = title.split(' / ', 2)[-1]
+        playlist_id = self._search_regex(r'nodeId:\s*(\d+)', webpage, 'playlist id')
+        playlist = self._force_download_webpage(
+            'http://vod.tvp.pl/vod/seriesAjax?type=series&nodeId=%s&recommend'
+            'edId=0&sort=&page=0&pageSize=1000000' % playlist_id, display_id)
+        videos_paths = re.findall(
+            '(?s)class="shortTitle">.*?href="(/[^"]+)', playlist)
+        entries = [
+            self.url_result('http://vod.tvp.pl%s' % v_path, ie=TvpIE.ie_key())
+            for v_path in videos_paths]
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'display_id': display_id,
+            'title': title,
+            'entries': entries,
+        }

From b92c54869345b60a29882297ac1832429f964d03 Mon Sep 17 00:00:00 2001
From: Zack Fernandes <zack@ohno.io>
Date: Wed, 3 Dec 2014 20:22:48 -0800
Subject: [PATCH 056/974] [foxgay] Initial support

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/foxgay.py   | 48 ++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 youtube_dl/extractor/foxgay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8b513ffd1..b2c7fd586 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -121,6 +121,7 @@ from .fktv import (
 from .flickr import FlickrIE
 from .folketinget import FolketingetIE
 from .fourtube import FourTubeIE
+from .foxgay import FoxgayIE
 from .franceculture import FranceCultureIE
 from .franceinter import FranceInterIE
 from .francetv import (
diff --git a/youtube_dl/extractor/foxgay.py b/youtube_dl/extractor/foxgay.py
new file mode 100644
index 000000000..a146a9180
--- /dev/null
+++ b/youtube_dl/extractor/foxgay.py
@@ -0,0 +1,48 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class FoxgayIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?foxgay\.com/videos/(?:\S+-)?(?P<id>\d+)\.shtml'
+    _TEST = {
+        'url': 'http://foxgay.com/videos/fuck-turkish-style-2582.shtml',
+        'md5': '80d72beab5d04e1655a56ad37afe6841',
+        'info_dict': {
+            'id': '2582',
+            'ext': 'mp4',
+            'title': 'md5:6122f7ae0fc6b21ebdf59c5e083ce25a',
+            'description': 'md5:5e51dc4405f1fd315f7927daed2ce5cf',
+            'age_limit': 18,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # Download the initial webpage and extract all available data.
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(
+            r'<title>(?P<title>.*?)</title>',
+            webpage, 'title', fatal=False)
+        description = self._html_search_regex(
+            r'<div class="ico_desc"><h2>(?P<description>.*?)</h2>',
+            webpage, 'description', fatal=False)
+
+        # Find the URL for the iFrame which contains the actual video.
+        iframe = self._download_webpage(
+            self._html_search_regex(r'iframe src="(?P<frame>.*?)"', webpage, 'video frame'),
+            video_id)
+        video_url = self._html_search_regex(
+            r"v_path = '(?P<vid>http://.*?)'", iframe, 'url')
+        thumb_url = self._html_search_regex(
+            r"t_path = '(?P<thumb>http://.*?)'", iframe, 'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'description': description,
+            'thumbnail': thumb_url,
+            'age_limit': 18,
+        }

From ecd7ea1e6bb5e91dc3a4391fbd49c85dab337bd9 Mon Sep 17 00:00:00 2001
From: Zack Fernandes <zack@ohno.io>
Date: Wed, 3 Dec 2014 22:22:36 -0800
Subject: [PATCH 057/974] [myvidster] Added support for Myvidster

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/myvidster.py | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 youtube_dl/extractor/myvidster.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8b513ffd1..f4f66fb0b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -245,6 +245,7 @@ from .muzu import MuzuTVIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvideo import MyVideoIE
+from .myvidster import MyVidsterIE
 from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
diff --git a/youtube_dl/extractor/myvidster.py b/youtube_dl/extractor/myvidster.py
new file mode 100644
index 000000000..7dd36ee46
--- /dev/null
+++ b/youtube_dl/extractor/myvidster.py
@@ -0,0 +1,28 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MyVidsterIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?myvidster\.com/video/(?P<id>\d+)/.*$'
+
+    _TEST = {
+        'url': 'http://www.myvidster.com/video/32059805/Hot_chemistry_with_raw_love_making',
+        'md5': '95296d0231c1363222c3441af62dc4ca',
+        'info_dict': {
+            'id': '3685814',
+            'title': 'md5:7d8427d6d02c4fbcef50fe269980c749',
+            'upload_date': '20141027',
+            'uploader_id': 'utkualp',
+            'ext': 'mp4',
+        },
+        'add_ie': ['XHamster'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        return self.url_result(self._html_search_regex(
+            r'rel="videolink" href="(?P<real_url>.*)">',
+            webpage, 'real video url'))

From d4e06d4a83b223e82814cd160bdf62e3a82f33e4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 07:57:18 +0100
Subject: [PATCH 058/974] [options] Standardize mentoined configuration file
 location (Fixes #4367)

---
 youtube_dl/options.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2e8c71508..041ca83d8 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -163,7 +163,10 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '--ignore-config',
         action='store_true',
-        help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
+        help='Do not read configuration files. '
+        'When given in the global configuration file /etc/youtube-dl.conf: '
+        'Do not read the user configuration in ~/.config/youtube-dl/config '
+        '(%APPDATA%/youtube-dl/config.txt on Windows)')
     general.add_option(
         '--flat-playlist',
         action='store_const', dest='extract_flat', const='in_playlist',

From abe70fa04440398f2262d34d7229cbb40db3e70f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 4 Dec 2014 08:21:29 +0100
Subject: [PATCH 059/974] [audiomack] Modernize test definition

---
 youtube_dl/extractor/audiomack.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 04386f7f7..9ce4982d9 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -26,9 +26,10 @@ class AudiomackIE(InfoExtractor):
         {
             'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
-            'file': '172419696.mp3',
             'info_dict':
             {
+                'id': '172419696',
+                'ext': 'mp3',
                 'description': 'md5:1fc3272ed7a635cce5be1568c2822997',
                 'title': 'Young Thug ft Lil Wayne - Take Kare',
                 'uploader':'Young Thug World',

From 191cc41ba49a5d19e798131060c6976dbb2d357e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:22:20 +0100
Subject: [PATCH 060/974] [foxgay] Add thumbnail to test definition

---
 youtube_dl/extractor/foxgay.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/foxgay.py b/youtube_dl/extractor/foxgay.py
index a146a9180..08b8ea362 100644
--- a/youtube_dl/extractor/foxgay.py
+++ b/youtube_dl/extractor/foxgay.py
@@ -14,14 +14,14 @@ class FoxgayIE(InfoExtractor):
             'title': 'md5:6122f7ae0fc6b21ebdf59c5e083ce25a',
             'description': 'md5:5e51dc4405f1fd315f7927daed2ce5cf',
             'age_limit': 18,
+            'thumbnail': 're:https?://.*\.jpg$',
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
-        # Download the initial webpage and extract all available data.
         webpage = self._download_webpage(url, video_id)
+
         title = self._html_search_regex(
             r'<title>(?P<title>.*?)</title>',
             webpage, 'title', fatal=False)

From 5f5e993dc61e68402d60f35df66a08b8bcca808c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:22:53 +0100
Subject: [PATCH 061/974] [bbccouk] Remove unused import

---
 youtube_dl/extractor/bbccouk.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index beb6cfc8a..01c02d360 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import re
 import xml.etree.ElementTree
 
 from .subtitles import SubtitlesInfoExtractor

From 810fb84d5edaccfdebbf1c4e441d7747f2688b7b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:27:40 +0100
Subject: [PATCH 062/974] pep8 and minor beautification all around

---
 youtube_dl/extractor/audiomack.py | 10 +++++-----
 youtube_dl/extractor/common.py    |  3 ++-
 youtube_dl/extractor/myspace.py   |  1 +
 youtube_dl/extractor/udemy.py     |  5 +----
 youtube_dl/extractor/youtube.py   | 13 ++++++++-----
 5 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 04386f7f7..622b20989 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -26,13 +26,13 @@ class AudiomackIE(InfoExtractor):
         {
             'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
-            'file': '172419696.mp3',
-            'info_dict':
-            {
+            'info_dict': {
+                'id': '172419696',
+                'ext': 'mp3',
                 'description': 'md5:1fc3272ed7a635cce5be1568c2822997',
                 'title': 'Young Thug ft Lil Wayne - Take Kare',
-                'uploader':'Young Thug World',
-                'upload_date':'20141016',
+                'uploader': 'Young Thug World',
+                'upload_date': '20141016',
             }
         },
     ]
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e80a2dad0..5b5e33cea 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -819,7 +819,8 @@ class InfoExtractor(object):
         return res
 
     def _set_cookie(self, domain, name, value, expire_time=None):
-        cookie = compat_cookiejar.Cookie(0, name, value, None, None, domain, None,
+        cookie = compat_cookiejar.Cookie(
+            0, name, value, None, None, domain, None,
             None, '/', True, False, expire_time, '', None, None, None)
         self._downloader.cookiejar.set_cookie(cookie)
 
diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index e62614670..83414a232 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -88,6 +88,7 @@ class MySpaceIE(InfoExtractor):
                 self.report_warning(
                     '%s: No downloadable song on this page' % video_id)
                 return
+
             def search_data(name):
                 return self._search_regex(
                     r'''data-%s=([\'"])(?P<data>.*?)\1''' % name,
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 0e4d386a8..5271611ac 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -97,11 +97,8 @@ class UdemyIE(InfoExtractor):
         if 'returnUrl' not in response:
             raise ExtractorError('Unable to log in')
 
-
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        lecture_id = mobj.group('id')
+        lecture_id = self._match_id(url)
 
         lecture = self._download_json(
             'https://www.udemy.com/api-1.1/lectures/%s' % lecture_id,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1cba40387..f452a90d8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -44,9 +44,10 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     _LOGIN_REQUIRED = False
 
     def _set_language(self):
-        self._set_cookie('.youtube.com', 'PREF', 'f1=50000000&hl=en',
+        self._set_cookie(
+            '.youtube.com', 'PREF', 'f1=50000000&hl=en',
             # YouTube sets the expire time to about two months
-            expire_time=time.time() + 60*24*3600)
+            expire_time=time.time() + 2 * 30 * 24 * 3600)
 
     def _login(self):
         """
@@ -722,9 +723,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 # We fallback to the get_video_info pages (used by the embed page)
                 self.report_video_info_webpage_download(video_id)
                 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                    video_info_url = (proto + '://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                        % (video_id, el_type))
-                    video_info_webpage = self._download_webpage(video_info_url,
+                    video_info_url = (
+                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
+                        % (proto, video_id, el_type))
+                    video_info_webpage = self._download_webpage(
+                        video_info_url,
                         video_id, note=False,
                         errnote='unable to download video info webpage')
                     video_info = compat_parse_qs(video_info_webpage)

From 8b97115358c89063cef27edc26f39cf64cbe4a87 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:28:41 +0100
Subject: [PATCH 063/974] Credit @zackfern for foxgay (#4371)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bd2e967e3..4d3b9ddc1 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -89,3 +89,4 @@ Oskar Jauch
 Matthew Rayfield
 t0mm0
 Tithen-Firion
+Zack Fernandes

From 39c52bbd32efa5367eae31af59066f045bc7fe7e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:31:55 +0100
Subject: [PATCH 064/974] [myvidster] Enforce age limit in test

---
 youtube_dl/extractor/myvidster.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myvidster.py b/youtube_dl/extractor/myvidster.py
index 7dd36ee46..a94ab8358 100644
--- a/youtube_dl/extractor/myvidster.py
+++ b/youtube_dl/extractor/myvidster.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class MyVidsterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?myvidster\.com/video/(?P<id>\d+)/.*$'
+    _VALID_URL = r'http://(?:www\.)?myvidster\.com/video/(?P<id>\d+)/'
 
     _TEST = {
         'url': 'http://www.myvidster.com/video/32059805/Hot_chemistry_with_raw_love_making',
@@ -15,6 +15,7 @@ class MyVidsterIE(InfoExtractor):
             'upload_date': '20141027',
             'uploader_id': 'utkualp',
             'ext': 'mp4',
+            'age_limit': 18,
         },
         'add_ie': ['XHamster'],
     }

From 9776bc7f57f061d133b204c056b1cebee775ddad Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:34:12 +0100
Subject: [PATCH 065/974] release 2014.12.04

---
 README.md             | 8 ++++----
 youtube_dl/version.py | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index d6e7ff902..18be078d7 100644
--- a/README.md
+++ b/README.md
@@ -65,10 +65,10 @@ which means you can modify it, redistribute it or use it however you like.
                                      this is not possible instead of searching.
     --ignore-config                  Do not read configuration files. When given
                                      in the global configuration file /etc
-                                     /youtube-dl.conf: do not read the user
-                                     configuration in ~/.config/youtube-dl.conf
-                                     (%APPDATA%/youtube-dl/config.txt on
-                                     Windows)
+                                     /youtube-dl.conf: Do not read the user
+                                     configuration in ~/.config/youtube-
+                                     dl/config (%APPDATA%/youtube-dl/config.txt
+                                     on Windows)
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them.
 
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 61902a8cc..7ff8633fe 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.03'
+__version__ = '2014.12.04'

From 225e4b9633285b66adc914a61d8f55ca125eb91d Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 4 Dec 2014 14:08:08 +0100
Subject: [PATCH 066/974] [tvp] Remove unnecessary code

---
 youtube_dl/extractor/tvp.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 2248a9fdf..bd7ee25a0 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -79,15 +79,8 @@ class TvpIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'description': self._og_search_description(webpage, default=''),
+            'description': self._og_search_description(webpage),
         }
-        if mobj.group('type') == 'vod' and info_dict['description'] == '':
-            info_dict.update({
-                'description': self._html_search_regex(
-                    r'(?s)<div\s+class=[\'"]opis.*?</div>',
-                    self._download_webpage(url, video_id), 'description', group=0),
-            })
-
         video_url = self._search_regex(
             r'0:{src:([\'"])(?P<url>.*?)\1', webpage, 'formats', group='url', default=None)
         if video_url is None:

From 995ad69c54899a0cfc84fd89083f07919acdbb83 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 4 Dec 2014 14:11:27 +0100
Subject: [PATCH 067/974] [common] Add new parameters for _download_webpage

---
 youtube_dl/extractor/common.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e80a2dad0..b633ea9b9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -360,9 +360,19 @@ class InfoExtractor(object):
 
         return content
 
-    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
+    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5):
         """ Returns the data of the page as a string """
-        res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
+        success = False
+        try_count = 0
+        while success is False:
+            try:
+                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
+                success = True
+            except compat_http_client.IncompleteRead as e:
+                try_count += 1
+                if try_count >= tries:
+                    raise e
+                self._sleep(timeout, video_id)
         if res is False:
             return res
         else:

From 2415951ead0c4996d7892ef859d06970c4595701 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 4 Dec 2014 14:12:09 +0100
Subject: [PATCH 068/974] [tvp] Modernize

---
 youtube_dl/extractor/tvp.py | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index bd7ee25a0..cf21d1202 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -127,26 +127,15 @@ class TvpSeriesIE(InfoExtractor):
         }
     ]
 
-    def _force_download_webpage(self, url, v_id, tries=0):
-        if tries >= 5:
-            raise ExtractorError(
-                '%s: Cannot download webpage, try again later' % v_id)
-        # Sometimes happen, but in my tests second try always succeeded
-        try:
-            return self._download_webpage(url, v_id)
-        except IncompleteRead as e:
-            return self._force_download_webpage(url, v_id, tries+1)
-    
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        webpage = self._force_download_webpage(url, display_id)
+        webpage = self._download_webpage(url, display_id, tries=5)
         title = self._html_search_regex(
-            r'(?s) id=[\'"]path[\'"]>(.*?)</span>', webpage, 'series')
-        title = title.split(' / ', 2)[-1]
+            r'(?s) id=[\'"]path[\'"]>(?:.*? / ){2}(.*?)</span>', webpage, 'series')
         playlist_id = self._search_regex(r'nodeId:\s*(\d+)', webpage, 'playlist id')
-        playlist = self._force_download_webpage(
+        playlist = self._download_webpage(
             'http://vod.tvp.pl/vod/seriesAjax?type=series&nodeId=%s&recommend'
-            'edId=0&sort=&page=0&pageSize=1000000' % playlist_id, display_id)
+            'edId=0&sort=&page=0&pageSize=10000' % playlist_id, display_id, tries=5)
         videos_paths = re.findall(
             '(?s)class="shortTitle">.*?href="(/[^"]+)', playlist)
         entries = [

From b128c9ed68394c5c6dd1c46f81d21314ed03ac18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Dec 2014 20:12:06 +0600
Subject: [PATCH 069/974] [vine:user] Add support for another URL format
 (Closes #4365)

---
 youtube_dl/extractor/vine.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 42995226e..4970b2f23 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -63,29 +63,36 @@ class VineIE(InfoExtractor):
 
 class VineUserIE(InfoExtractor):
     IE_NAME = 'vine:user'
-    _VALID_URL = r'(?:https?://)?vine\.co/(?P<user>[^/]+)/?(\?.*)?$'
+    _VALID_URL = r'(?:https?://)?vine\.co/(?P<u>u/)?(?P<user>[^/]+)/?(\?.*)?$'
     _VINE_BASE_URL = "https://vine.co/"
-    _TEST = {
-        'url': 'https://vine.co/Visa',
-        'info_dict': {
-            'id': 'Visa',
+    _TESTS = [
+        {
+            'url': 'https://vine.co/Visa',
+            'info_dict': {
+                'id': 'Visa',
+            },
+            'playlist_mincount': 46,
         },
-        'playlist_mincount': 46,
-    }
+        {
+            'url': 'https://vine.co/u/941705360593584128',
+            'only_matching': True,
+        },
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         user = mobj.group('user')
+        u = mobj.group('u')
 
-        profile_url = "%sapi/users/profiles/vanity/%s" % (
-            self._VINE_BASE_URL, user)
+        profile_url = "%sapi/users/profiles/%s%s" % (
+            self._VINE_BASE_URL, 'vanity/' if not u else '', user)
         profile_data = self._download_json(
             profile_url, user, note='Downloading user profile data')
 
         user_id = profile_data['data']['userId']
         timeline_data = []
         for pagenum in itertools.count(1):
-            timeline_url = "%sapi/timelines/users/%s?page=%s" % (
+            timeline_url = "%sapi/timelines/users/%s?page=%s&size=100" % (
                 self._VINE_BASE_URL, user_id, pagenum)
             timeline_page = self._download_json(
                 timeline_url, user, note='Downloading page %d' % pagenum)

From 1139a54d9bb9d214a8451301a44af6d03dd450b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Dec 2014 21:19:08 +0600
Subject: [PATCH 070/974] [foxnews] Add extractor (Closes #4352)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/foxnews.py  | 94 ++++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)
 create mode 100644 youtube_dl/extractor/foxnews.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 647352b59..0a5151d5a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -122,6 +122,7 @@ from .flickr import FlickrIE
 from .folketinget import FolketingetIE
 from .fourtube import FourTubeIE
 from .foxgay import FoxgayIE
+from .foxnews import FoxNewsIE
 from .franceculture import FranceCultureIE
 from .franceinter import FranceInterIE
 from .francetv import (
diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
new file mode 100644
index 000000000..917f76b1e
--- /dev/null
+++ b/youtube_dl/extractor/foxnews.py
@@ -0,0 +1,94 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    int_or_none,
+)
+
+
+class FoxNewsIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.foxnews\.com/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://video.foxnews.com/v/3937480/frozen-in-time/#sp=show-clips',
+            'md5': '32aaded6ba3ef0d1c04e238d01031e5e',
+            'info_dict': {
+                'id': '3937480',
+                'ext': 'flv',
+                'title': 'Frozen in Time',
+                'description': 'Doctors baffled by 16-year-old girl that is the size of a toddler',
+                'duration': 265,
+                'timestamp': 1304411491,
+                'upload_date': '20110503',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+        },
+        {
+            'url': 'http://video.foxnews.com/v/3922535568001/rep-luis-gutierrez-on-if-obamas-immigration-plan-is-legal/#sp=show-clips',
+            'md5': '5846c64a1ea05ec78175421b8323e2df',
+            'info_dict': {
+                'id': '3922535568001',
+                'ext': 'mp4',
+                'title': "Rep. Luis Gutierrez on if Obama's immigration plan is legal",
+                'description': "Congressman discusses the president's executive action",
+                'duration': 292,
+                'timestamp': 1417662047,
+                'upload_date': '20141204',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+        },
+        {
+            'url': 'http://video.foxnews.com/v/video-embed.html?video_id=3937480&d=video.foxnews.com',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video = self._download_json(
+            'http://video.foxnews.com/v/feed/video/%s.js?template=fox' % video_id, video_id)
+
+        item = video['channel']['item']
+        title = item['title']
+        description = item['description']
+        timestamp = parse_iso8601(item['dc-date'])
+
+        media_group = item['media-group']
+        duration = None
+        formats = []
+        for media in media_group['media-content']:
+            attributes = media['@attributes']
+            video_url = attributes['url']
+            if video_url.endswith('.f4m'):
+                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124', video_id))
+            elif video_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, 'flv'))
+            elif not video_url.endswith('.smil'):
+                duration = int_or_none(attributes.get('duration'))
+                formats.append({
+                    'url': video_url,
+                    'format_id': media['media-category']['@attributes']['label'],
+                    'preference': 1,
+                    'vbr': int_or_none(attributes.get('bitrate')),
+                    'filesize': int_or_none(attributes.get('fileSize'))
+                })
+        self._sort_formats(formats)
+
+        media_thumbnail = media_group['media-thumbnail']['@attributes']
+        thumbnails = [{
+            'url': media_thumbnail['url'],
+            'width': int_or_none(media_thumbnail.get('width')),
+            'height': int_or_none(media_thumbnail.get('height')),
+        }] if media_thumbnail else []
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }

From 4349c07dd7cd07620365b36093f0a148c41ce434 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 17:02:05 +0100
Subject: [PATCH 071/974] [minhateca] Add extractor (Fixes #4094)

---
 test/test_utils.py                |  1 +
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/minhateca.py | 71 +++++++++++++++++++++++++++++++
 youtube_dl/utils.py               |  7 ++-
 4 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/minhateca.py

diff --git a/test/test_utils.py b/test/test_utils.py
index baa3a2156..04f1bf283 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -376,6 +376,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_filesize('2 MiB'), 2097152)
         self.assertEqual(parse_filesize('5 GB'), 5000000000)
         self.assertEqual(parse_filesize('1.2Tb'), 1200000000000)
+        self.assertEqual(parse_filesize('1,24 KB'), 1240)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 647352b59..a56ec4fb5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -217,6 +217,7 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
+from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mitele import MiTeleIE
diff --git a/youtube_dl/extractor/minhateca.py b/youtube_dl/extractor/minhateca.py
new file mode 100644
index 000000000..077c9b19d
--- /dev/null
+++ b/youtube_dl/extractor/minhateca.py
@@ -0,0 +1,71 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
+)
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+)
+
+
+class MinhatecaIE(InfoExtractor):
+    _VALID_URL = r'https?://minhateca\.com\.br/[^?#]+,(?P<id>[0-9]+)\.'
+    _TEST = {
+        'url': 'http://minhateca.com.br/pereba/misc/youtube-dl+test+video,125848331.mp4(video)',
+        'info_dict': {
+            'id': '125848331',
+            'ext': 'mp4',
+            'title': 'youtube-dl test video',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'filesize_approx': 1530000,
+            'duration': 9,
+            'view_count': int,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        token = self._html_search_regex(
+            r'<input name="__RequestVerificationToken".*?value="([^"]+)"',
+            webpage, 'request token')
+        token_data = [
+            ('fileId', video_id),
+            ('__RequestVerificationToken', token),
+        ]
+        req = compat_urllib_request.Request(
+            'http://minhateca.com.br/action/License/Download',
+            data=compat_urllib_parse.urlencode(token_data))
+        req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        data = self._download_json(
+            req, video_id, note='Downloading metadata')
+
+        video_url = data['redirectUrl']
+        title_str = self._html_search_regex(
+            r'<h1.*?>(.*?)</h1>', webpage, 'title')
+        title, _, ext = title_str.rpartition('.')
+        filesize_approx = parse_filesize(self._html_search_regex(
+            r'<p class="fileSize">(.*?)</p>',
+            webpage, 'file size approximation', fatal=False))
+        duration = int_or_none(self._html_search_regex(
+            r'(?s)<p class="fileLeng[ht][th]">.*?([0-9]+)\s*s',
+            webpage, 'duration', fatal=False))
+        view_count = int_or_none(self._html_search_regex(
+            r'<p class="downloadsCounter">([0-9]+)</p>',
+            webpage, 'view count', fatal=False))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': ext,
+            'filesize_approx': filesize_approx,
+            'duration': duration,
+            'view_count': view_count,
+            'thumbnail': self._og_search_thumbnail(webpage),
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4d3cbac74..5e9ae7a42 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1090,11 +1090,14 @@ def parse_filesize(s):
     }
 
     units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
-    m = re.match(r'(?P<num>[0-9]+(?:\.[0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
+    m = re.match(
+        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
     if not m:
         return None
 
-    return int(float(m.group('num')) * _UNIT_TABLE[m.group('unit')])
+    num_str = m.group('num').replace(',', '.')
+    mult = _UNIT_TABLE[m.group('unit')]
+    return int(float(num_str) * mult)
 
 
 def get_term_width():

From ab07963b5cc79812c6fb7e4f9e363533d8123830 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 17:02:23 +0100
Subject: [PATCH 072/974] release 2014.12.04.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7ff8633fe..cce669059 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.04'
+__version__ = '2014.12.04.1'

From e8df5cee12378acd708b6686130a73c5edc06f0e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 17:35:40 +0100
Subject: [PATCH 073/974] [minhateca] Fix duration parsing

---
 test/test_utils.py                |  3 +++
 youtube_dl/extractor/minhateca.py |  5 +++--
 youtube_dl/utils.py               | 19 +++++++++++++++----
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 04f1bf283..badab1370 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -220,6 +220,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('0s'), 0)
         self.assertEqual(parse_duration('01:02:03.05'), 3723.05)
         self.assertEqual(parse_duration('T30M38S'), 1838)
+        self.assertEqual(parse_duration('5 s'), 5)
+        self.assertEqual(parse_duration('3 min'), 180)
+        self.assertEqual(parse_duration('2.5 hours'), 9000)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/extractor/minhateca.py b/youtube_dl/extractor/minhateca.py
index 077c9b19d..14934b7ec 100644
--- a/youtube_dl/extractor/minhateca.py
+++ b/youtube_dl/extractor/minhateca.py
@@ -8,6 +8,7 @@ from ..compat import (
 )
 from ..utils import (
     int_or_none,
+    parse_duration,
     parse_filesize,
 )
 
@@ -52,8 +53,8 @@ class MinhatecaIE(InfoExtractor):
         filesize_approx = parse_filesize(self._html_search_regex(
             r'<p class="fileSize">(.*?)</p>',
             webpage, 'file size approximation', fatal=False))
-        duration = int_or_none(self._html_search_regex(
-            r'(?s)<p class="fileLeng[ht][th]">.*?([0-9]+)\s*s',
+        duration = parse_duration(self._html_search_regex(
+            r'(?s)<p class="fileLeng[ht][th]">.*?class="bold">(.*?)<',
             webpage, 'duration', fatal=False))
         view_count = int_or_none(self._html_search_regex(
             r'<p class="downloadsCounter">([0-9]+)</p>',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5e9ae7a42..5efb4c59a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1206,18 +1206,29 @@ def parse_duration(s):
 
     m = re.match(
         r'''(?ix)T?
+        (?:
+            (?P<only_mins>[0-9.]+)\s*(?:mins?|minutes?)\s*|
+            (?P<only_hours>[0-9.]+)\s*(?:hours?)|
+
             (?:
                 (?:(?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*)?
                 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
             )?
-            (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?$''', s)
+            (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
+        )$''', s)
     if not m:
         return None
-    res = int(m.group('secs'))
+    res = 0
+    if m.group('only_mins'):
+        return float_or_none(m.group('only_mins'), invscale=60)
+    if m.group('only_hours'):
+        return float_or_none(m.group('only_hours'), invscale=60 * 60)
+    if m.group('secs'):
+        res += int(m.group('secs'))
     if m.group('mins'):
         res += int(m.group('mins')) * 60
-        if m.group('hours'):
-            res += int(m.group('hours')) * 60 * 60
+    if m.group('hours'):
+        res += int(m.group('hours')) * 60 * 60
     if m.group('ms'):
         res += float(m.group('ms'))
     return res

From 81028ff9ebdfdccc680d404ce49a3fa32ac965c0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 17:43:34 +0100
Subject: [PATCH 074/974] [xminus] Capture description (#4300)

---
 youtube_dl/extractor/xminus.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/xminus.py b/youtube_dl/extractor/xminus.py
index f7e2e8ac9..8c6241aed 100644
--- a/youtube_dl/extractor/xminus.py
+++ b/youtube_dl/extractor/xminus.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_chr,
@@ -25,6 +27,7 @@ class XMinusIE(InfoExtractor):
             'tbr': 320,
             'filesize_approx': 5900000,
             'view_count': int,
+            'description': 'md5:03238c5b663810bc79cf42ef3c03e371',
         }
     }
 
@@ -48,6 +51,11 @@ class XMinusIE(InfoExtractor):
         view_count = int_or_none(self._html_search_regex(
             r'<div class="quality.*?► ([0-9]+)',
             webpage, 'view count', fatal=False))
+        description = self._html_search_regex(
+            r'(?s)<div id="song_texts">(.*?)</div><br',
+            webpage, 'song lyrics', fatal=False)
+        if description:
+            description = re.sub(' *\r *', '\n', description)
 
         enc_token = self._html_search_regex(
             r'minus_track\.tkn="(.+?)"', webpage, 'enc_token')
@@ -64,4 +72,5 @@ class XMinusIE(InfoExtractor):
             'filesize_approx': filesize_approx,
             'tbr': tbr,
             'view_count': view_count,
+            'description': description,
         }

From a154eb3d159baa2b7f8f4dcd9c1215d7a3486651 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 17:43:39 +0100
Subject: [PATCH 075/974] release 2014.12.04.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cce669059..0f902b1e9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.04.1'
+__version__ = '2014.12.04.2'

From 122c2f87c17984e4ae04cbb3c3631b547dc61105 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 5 Dec 2014 10:59:55 +0100
Subject: [PATCH 076/974] [tagesschau] Modernize

---
 youtube_dl/extractor/tagesschau.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index b87047451..17e123112 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -28,14 +28,8 @@ class TagesschauIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        if video_id.startswith('-'):
-            display_id = video_id.strip('-')
-        else:
-            display_id = video_id
-
+        video_id = self._match_id(url)
+        display_id = video_id.lstrip('-')
         webpage = self._download_webpage(url, display_id)
 
         playerpage = self._download_webpage(

From ebb6419960eafbdde055d016b4f95620c724effb Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Fri, 5 Dec 2014 12:07:06 +0100
Subject: [PATCH 077/974] [common] Split _download_json Add ability for
 extractor to use _parse_json

---
 youtube_dl/extractor/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e80a2dad0..7cbd846f6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -390,6 +390,10 @@ class InfoExtractor(object):
             url_or_request, video_id, note, errnote, fatal=fatal)
         if (not fatal) and json_string is False:
             return None
+        return self._parse_json(
+            json_string, video_id, transform_source=transform_source, fatal=fatal)
+
+    def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
         if transform_source:
             json_string = transform_source(json_string)
         try:

From d958fa9ff961fb17b55aba0d70e5175e2c63f631 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Fri, 5 Dec 2014 00:09:17 +0100
Subject: [PATCH 078/974] [howstuffworks] Rewrite extractor

---
 youtube_dl/extractor/howstuffworks.py | 139 ++++++++------------------
 1 file changed, 40 insertions(+), 99 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index fccc23884..c46e8b911 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -1,134 +1,75 @@
 from __future__ import unicode_literals
 
-import re
-import json
-import random
-import string
-
 from .common import InfoExtractor
-from ..utils import find_xpath_attr
 
 
 class HowStuffWorksIE(InfoExtractor):
     _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*\d+-(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
-            'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',
+            'url': 'http://adventure.howstuffworks.com/39521-deadliest-catch-nautical-collision-video.htm',
             'info_dict': {
-                'id': '450221',
-                'display_id': 'cool-jobs-iditarod-musher',
-                'ext': 'flv',
-                'title': 'Cool Jobs - Iditarod Musher',
-                'description': 'md5:82bb58438a88027b8186a1fccb365f90',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'id': '553475',
+                'ext': 'mp4',
+                'title': 'Deadliest Catch: Nautical Collision',
+                'description': 'Check out this clip to get an exclusive look at the Deadliest Catch crew back in action.',
+                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/39521.jpg',
             },
-            'params': {
-                # md5 is not consistent
-                'skip_download': True
-            }
         },
         {
             'url': 'http://adventure.howstuffworks.com/7199-survival-zone-food-and-water-in-the-savanna-video.htm',
             'info_dict': {
                 'id': '453464',
-                'display_id': 'survival-zone-food-and-water-in-the-savanna',
                 'ext': 'mp4',
                 'title': 'Survival Zone: Food and Water In the Savanna',
-                'description': 'md5:7e1c89f6411434970c15fa094170c371',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'Learn how to find both food and water while trekking in the African savannah. In this video from the Discovery Channel.',
+                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/7199.jpg',
             },
-            'params': {
-                # md5 is not consistent
-                'skip_download': True
-            }
         },
         {
-            'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm',
+            'url': 'http://entertainment.howstuffworks.com/arts/34476-time-warp-fire-breathing-disaster-video.htm',
             'info_dict': {
-                'id': '440011',
-                'display_id': 'sword-swallowing-1-by-dan-meyer',
-                'ext': 'flv',
-                'title': 'Sword Swallowing #1 by Dan Meyer',
-                'description': 'md5:b2409e88172913e2e7d3d1159b0ef735',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'id': '487036',
+                'ext': 'mp4',
+                'title': 'Time Warp: Fire-Breathing Disaster',
+                'description': 'Fire-breathers are safe from flames when they angle their heads up as they blow -- see what would happen if they looked down instead. Check out this clip from Discovery\'s "Time Warp" series to learn more.',
+                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/34476.jpg',
             },
-            'params': {
-                # md5 is not consistent
-                'skip_download': True
-            }
         },
     ]
 
+    def _extract_clip_info(self, key, clip_info, name=None, **kargs):
+        if name is None:
+            name = key
+        return self._html_search_regex(
+            r"\s*%s\s*: '?([^'\n]*[^'\n,])" % key, clip_info, name, **kargs)
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
+        display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-
-        content_id = self._search_regex(r'var siteSectionId="(\d+)";', webpage, 'content id')
-
-        mp4 = self._search_regex(
-            r'''(?xs)var\s+clip\s*=\s*{\s*
-                .+?\s*
-                content_id\s*:\s*%s\s*,\s*
-                .+?\s*
-                mp4\s*:\s*\[(.*?),?\]\s*
-                };\s*
-                videoData\.push\(clip\);''' % content_id,
-            webpage, 'mp4', fatal=False, default=None)
-
-        smil = self._download_xml(
-            'http://services.media.howstuffworks.com/videos/%s/smil-service.smil' % content_id,
-            content_id, 'Downloading video SMIL')
-
-        http_base = find_xpath_attr(
-            smil,
-            './{0}head/{0}meta'.format('{http://www.w3.org/2001/SMIL20/Language}'),
-            'name',
-            'httpBase').get('content')
-
-        def random_string(str_len=0):
-            return ''.join([random.choice(string.ascii_uppercase) for _ in range(str_len)])
-
-        URL_SUFFIX = '?v=2.11.3&fp=LNX 11,2,202,356&r=%s&g=%s' % (random_string(5), random_string(12))
-
+        clip_info = self._search_regex('(?s)var clip = {(.*?)};', webpage, 'clip info')
+        video_id = self._extract_clip_info('content_id', clip_info, 'video id')
         formats = []
-
-        if mp4:
-            for video in json.loads('[%s]' % mp4):
-                bitrate = video['bitrate']
-                fmt = {
-                    'url': video['src'].replace('http://pmd.video.howstuffworks.com', http_base) + URL_SUFFIX,
-                    'format_id': bitrate,
-                }
-                m = re.search(r'(?P<vbr>\d+)[Kk]', bitrate)
-                if m:
-                    fmt['vbr'] = int(m.group('vbr'))
-                formats.append(fmt)
-        else:
-            for video in smil.findall(
-                    './/{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')):
-                vbr = int(video.attrib['system-bitrate']) / 1000
-                formats.append({
-                    'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX),
-                    'format_id': '%dk' % vbr,
-                    'vbr': vbr,
-                })
-
+        m3u8_url = self._extract_clip_info('m3u8', clip_info, 'm3u8 url', default=None)
+        if m3u8_url is not None:
+            formats += self._extract_m3u8_formats(m3u8_url , video_id, 'mp4')
+        mp4 = self._parse_json(
+            self._extract_clip_info(
+                'mp4', clip_info, 'formats').replace('},]','}]'), video_id)
+        for video in mp4:
+            formats.append({
+                'url': video['src'],
+                'format_id': video['bitrate'],
+                'tbr': int(video['bitrate'].rstrip('k')),
+            })
         self._sort_formats(formats)
 
-        title = self._og_search_title(webpage)
-        TITLE_SUFFIX = ' : HowStuffWorks'
-        if title.endswith(TITLE_SUFFIX):
-            title = title[:-len(TITLE_SUFFIX)]
-
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-
         return {
-            'id': content_id,
+            'id': video_id,
             'display_id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
+            'title': self._extract_clip_info('clip_title', clip_info, 'title'),
+            'description': self._extract_clip_info('caption', clip_info, 'description', fatal=False),
+            'thumbnail': self._extract_clip_info('video_still_url', clip_info, 'thumbnail'),
+            'duration': self._extract_clip_info('duration', clip_info),
             'formats': formats,
         }

From 90644a68436e53e3ee74af53841c750549816cd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Dec 2014 22:08:30 +0600
Subject: [PATCH 079/974] [azubu] Add extractor (Closes #4379)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/azubu.py    | 93 ++++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+)
 create mode 100644 youtube_dl/extractor/azubu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e0c4a3486..4c9c44b31 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -24,6 +24,7 @@ from .arte import (
 )
 from .audiomack import AudiomackIE
 from .auengine import AUEngineIE
+from .azubu import AzubuIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
diff --git a/youtube_dl/extractor/azubu.py b/youtube_dl/extractor/azubu.py
new file mode 100644
index 000000000..0961d339f
--- /dev/null
+++ b/youtube_dl/extractor/azubu.py
@@ -0,0 +1,93 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import float_or_none
+
+
+class AzubuIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?azubu\.tv/[^/]+#!/play/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://www.azubu.tv/GSL#!/play/15575/2014-hot6-cup-last-big-match-ro8-day-1',
+            'md5': 'a88b42fcf844f29ad6035054bd9ecaf4',
+            'info_dict': {
+                'id': '15575',
+                'ext': 'mp4',
+                'title': '2014 HOT6 CUP LAST BIG MATCH Ro8 Day 1',
+                'description': 'md5:d06bdea27b8cc4388a90ad35b5c66c01',
+                'thumbnail': 're:^https?://.*\.jpe?g',
+                'timestamp': 1417523507.334,
+                'upload_date': '20141202',
+                'duration': 9988.7,
+                'uploader': 'GSL',
+                'uploader_id': 414310,
+                'view_count': int,
+            },
+        },
+        {
+            'url': 'http://www.azubu.tv/FnaticTV#!/play/9344/-fnatic-at-worlds-2014:-toyz---%22i-love-rekkles,-he-has-amazing-mechanics%22-',
+            'md5': 'b72a871fe1d9f70bd7673769cdb3b925',
+            'info_dict': {
+                'id': '9344',
+                'ext': 'mp4',
+                'title': 'Fnatic at Worlds 2014: Toyz - "I love Rekkles, he has amazing mechanics"',
+                'description': 'md5:4a649737b5f6c8b5c5be543e88dc62af',
+                'thumbnail': 're:^https?://.*\.jpe?g',
+                'timestamp': 1410530893.320,
+                'upload_date': '20140912',
+                'duration': 172.385,
+                'uploader': 'FnaticTV',
+                'uploader_id': 272749,
+                'view_count': int,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        data = self._download_json(
+            'http://www.azubu.tv/api/video/%s' % video_id, video_id)['data']
+
+        title = data['title'].strip()
+        description = data['description']
+        thumbnail = data['thumbnail']
+        view_count = data['view_count']
+        uploader = data['user']['username']
+        uploader_id = data['user']['id']
+
+        stream_params = json.loads(data['stream_params'])
+
+        timestamp = float_or_none(stream_params['creationDate'], 1000)
+        duration = float_or_none(stream_params['length'], 1000)
+
+        renditions = stream_params.get('renditions') or []
+        video = stream_params.get('FLVFullLength') or stream_params.get('videoFullLength')
+        if video:
+            renditions.append(video)
+
+        formats = [{
+            'url': fmt['url'],
+            'width': fmt['frameWidth'],
+            'height': fmt['frameHeight'],
+            'vbr': float_or_none(fmt['encodingRate'], 1000),
+            'filesize': fmt['size'],
+            'vcodec': fmt['videoCodec'],
+            'container': fmt['videoContainer'],
+        } for fmt in renditions if fmt['url']]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'view_count': view_count,
+            'formats': formats,
+        }

From e638e83662e5baa414b8d80145e46c1912f7998d Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Fri, 5 Dec 2014 19:46:49 +0100
Subject: [PATCH 080/974] [howstuffworks] Update extractor

---
 youtube_dl/extractor/howstuffworks.py | 77 ++++++++++++++++++---------
 1 file changed, 51 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index c46e8b911..9478e9e67 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -1,19 +1,20 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import find_xpath_attr
 
 
 class HowStuffWorksIE(InfoExtractor):
     _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*\d+-(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
-            'url': 'http://adventure.howstuffworks.com/39521-deadliest-catch-nautical-collision-video.htm',
+            'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',
             'info_dict': {
-                'id': '553475',
-                'ext': 'mp4',
-                'title': 'Deadliest Catch: Nautical Collision',
-                'description': 'Check out this clip to get an exclusive look at the Deadliest Catch crew back in action.',
-                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/39521.jpg',
+                'id': '450221',
+                'ext': 'flv',
+                'title': 'Cool Jobs - Iditarod Musher',
+                'description': 'Cold sleds, freezing temps and warm dog breath... an Iditarod musher\'s dream. Kasey-Dee Gardner jumps on a sled to find out what the big deal is.',
+                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/5266.jpg',
             },
         },
         {
@@ -27,49 +28,73 @@ class HowStuffWorksIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://entertainment.howstuffworks.com/arts/34476-time-warp-fire-breathing-disaster-video.htm',
+            'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm',
             'info_dict': {
-                'id': '487036',
-                'ext': 'mp4',
-                'title': 'Time Warp: Fire-Breathing Disaster',
-                'description': 'Fire-breathers are safe from flames when they angle their heads up as they blow -- see what would happen if they looked down instead. Check out this clip from Discovery\'s "Time Warp" series to learn more.',
-                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/34476.jpg',
+                'id': '440011',
+                'ext': 'flv',
+                'title': 'Sword Swallowing #1 by Dan Meyer',
+                'description': 'Video footage (1 of 3) used by permission of the owner Dan Meyer through Sword Swallowers Association International <www.swordswallow.org>',
+                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/118306353233.jpg',
             },
         },
     ]
 
-    def _extract_clip_info(self, key, clip_info, name=None, **kargs):
-        if name is None:
-            name = key
-        return self._html_search_regex(
-            r"\s*%s\s*: '?([^'\n]*[^'\n,])" % key, clip_info, name, **kargs)
-
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
         clip_info = self._search_regex('(?s)var clip = {(.*?)};', webpage, 'clip info')
-        video_id = self._extract_clip_info('content_id', clip_info, 'video id')
+
+        def extract_clip_info(key, clip_info, name=None, **kargs):
+            if name is None:
+                name = key
+            return self._html_search_regex(
+                r"\s*%s\s*: '?([^'\n]*[^'\n,])" % key, clip_info, name, **kargs)
+
+        video_id = extract_clip_info('content_id', clip_info, 'video id')
         formats = []
-        m3u8_url = self._extract_clip_info('m3u8', clip_info, 'm3u8 url', default=None)
+        m3u8_url = extract_clip_info('m3u8', clip_info, 'm3u8 url', default=None)
         if m3u8_url is not None:
             formats += self._extract_m3u8_formats(m3u8_url , video_id, 'mp4')
         mp4 = self._parse_json(
-            self._extract_clip_info(
+            extract_clip_info(
                 'mp4', clip_info, 'formats').replace('},]','}]'), video_id)
         for video in mp4:
             formats.append({
                 'url': video['src'],
                 'format_id': video['bitrate'],
-                'tbr': int(video['bitrate'].rstrip('k')),
+                'vbr': int(video['bitrate'].rstrip('k')),
             })
+
+        if not formats:
+            smil = self._download_xml(
+                'http://services.media.howstuffworks.com/videos/%s/smil-service.smil' % video_id,
+                video_id, 'Downloading video SMIL')
+
+            http_base = find_xpath_attr(
+                smil,
+                './{0}head/{0}meta'.format('{http://www.w3.org/2001/SMIL20/Language}'),
+                'name',
+                'httpBase').get('content')
+
+            URL_SUFFIX = '?v=2.11.3&fp=LNX 11,2,202,356&r=A&g=A'
+
+            for video in smil.findall(
+                    './{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')):
+                vbr = int(video.attrib['system-bitrate']) / 1000
+                formats.append({
+                    'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX),
+                    'format_id': '%dk' % vbr,
+                    'vbr': vbr,
+                })
+
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'display_id': display_id,
-            'title': self._extract_clip_info('clip_title', clip_info, 'title'),
-            'description': self._extract_clip_info('caption', clip_info, 'description', fatal=False),
-            'thumbnail': self._extract_clip_info('video_still_url', clip_info, 'thumbnail'),
-            'duration': self._extract_clip_info('duration', clip_info),
+            'title': extract_clip_info('clip_title', clip_info, 'title'),
+            'description': extract_clip_info('caption', clip_info, 'description', fatal=False),
+            'thumbnail': extract_clip_info('video_still_url', clip_info, 'thumbnail'),
+            'duration': extract_clip_info('duration', clip_info),
             'formats': formats,
         }

From 045c48847ad066fc273337c96b25eacc35d146cd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 00:42:39 +0100
Subject: [PATCH 081/974] [tagesschau] Add suppot for sendung (Fixes #4378)

---
 youtube_dl/extractor/tagesschau.py | 101 ++++++++++++++++++++++-------
 1 file changed, 79 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 17e123112..aed670941 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -4,10 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import parse_filesize
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/video/video(?P<id>-?[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/ts|video/video)(?P<id>-?[0-9]+)\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
@@ -19,6 +20,16 @@ class TagesschauIE(InfoExtractor):
             'description': 'md5:69da3c61275b426426d711bde96463ab',
             'thumbnail': 're:^http:.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/ts-5727.html',
+        'md5': '3c54c1f6243d279b706bde660ceec633',
+        'info_dict': {
+            'id': '5727',
+            'ext': 'mp4',
+            'description': 'md5:695c01bfd98b7e313c501386327aea59',
+            'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
     }]
 
     _FORMATS = {
@@ -32,32 +43,78 @@ class TagesschauIE(InfoExtractor):
         display_id = video_id.lstrip('-')
         webpage = self._download_webpage(url, display_id)
 
-        playerpage = self._download_webpage(
-            'http://www.tagesschau.de/multimedia/video/video%s~player_autoplay-true.html' % video_id,
-            display_id, 'Downloading player page')
+        player_url = self._html_search_meta(
+            'twitter:player', webpage, 'player URL', default=None)
+        if player_url:
+            playerpage = self._download_webpage(
+                player_url, display_id, 'Downloading player page')
 
-        medias = re.findall(
-            r'"(http://media.+?)", type:"video/(.+?)", quality:"(.+?)"',
-            playerpage)
-
-        formats = []
-        for url, ext, res in medias:
-            f = {
-                'format_id': res + '_' + ext,
-                'url': url,
-                'ext': ext,
-            }
-            f.update(self._FORMATS.get(res, {}))
-            formats.append(f)
+            medias = re.findall(
+                r'"(http://media.+?)", type:"video/(.+?)", quality:"(.+?)"',
+                playerpage)
+            formats = []
+            for url, ext, res in medias:
+                f = {
+                    'format_id': res + '_' + ext,
+                    'url': url,
+                    'ext': ext,
+                }
+                f.update(self._FORMATS.get(res, {}))
+                formats.append(f)
+            thumbnail_fn = re.findall(r'"(/multimedia/.+?\.jpg)"', playerpage)[-1]
+            title = self._og_search_title(webpage).strip()
+            description = self._og_search_description(webpage).strip()
+        else:
+            download_text = self._search_regex(
+                r'(?s)<p>Wir bieten dieses Video in folgenden Formaten zum Download an:</p>\s*<div class="controls">(.*?)</div>\s*<p>',
+                webpage, 'download links')
+            links = re.finditer(
+                r'<div class="button" title="(?P<title>[^"]*)"><a href="(?P<url>[^"]+)">(?P<name>.+?)</a></div>',
+                webpage)
+            formats = []
+            for l in links:
+                format_id = self._search_regex(
+                    r'.*/[^/.]+\.([^/]+)\.[^/.]+', l.group('url'), 'format ID')
+                format = {
+                    'format_id': format_id,
+                    'url': l.group('url'),
+                    'format_name': l.group('name'),
+                }
+                m = re.match(
+                    r'''(?x)
+                        Video:\s*(?P<vcodec>[a-zA-Z0-9/._-]+)\s*&\#10;
+                        (?P<width>[0-9]+)x(?P<height>[0-9]+)px&\#10;
+                        (?P<vbr>[0-9]+)kbps&\#10;
+                        Audio:\s*(?P<abr>[0-9]+)kbps,\s*(?P<audio_desc>[A-Za-z\.0-9]+)&\#10;
+                        Gr&ouml;&szlig;e:\s*(?P<filesize_approx>[0-9.,]+\s+[a-zA-Z]*B)''',
+                    l.group('title'))
+                if m:
+                    format.update({
+                        'format_note': m.group('audio_desc'),
+                        'vcodec': m.group('vcodec'),
+                        'width': int(m.group('width')),
+                        'height': int(m.group('height')),
+                        'abr': int(m.group('abr')),
+                        'vbr': int(m.group('vbr')),
+                        'filesize_approx': parse_filesize(m.group('filesize_approx')),
+                    })
+                formats.append(format)
+            thumbnail_fn = self._search_regex(
+                r'(?s)<img alt="Sendungsbild".*?src="([^"]+)"',
+                webpage, 'thumbnail', fatal=False)
+            description = self._html_search_regex(
+                r'(?s)<p class="teasertext">(.*?)</p>',
+                webpage, 'description', fatal=False)
+            title = self._html_search_regex(
+                r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 
         self._sort_formats(formats)
-
-        thumbnail = re.findall(r'"(/multimedia/.+?\.jpg)"', playerpage)[-1]
+        thumbnail = 'http://www.tagesschau.de' + thumbnail_fn
 
         return {
             'id': display_id,
-            'title': self._og_search_title(webpage).strip(),
-            'thumbnail': 'http://www.tagesschau.de' + thumbnail,
+            'title': title,
+            'thumbnail': thumbnail,
             'formats': formats,
-            'description': self._og_search_description(webpage).strip(),
+            'description': description,
         }

From 1a7c6c69d3a1f16507b9fff87257cd7f2fef7f25 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 00:43:04 +0100
Subject: [PATCH 082/974] release 2014.12.06

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0f902b1e9..c568920d9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.04.2'
+__version__ = '2014.12.06'

From 58daf5ebed87fb92d0b4866777d448e1142e346b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 00:48:29 +0100
Subject: [PATCH 083/974] [youporn] Fix JSON parameter regexp (Fixes #4384)

---
 youtube_dl/extractor/youporn.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 8123928be..d9c06a2ee 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -45,7 +45,9 @@ class YouPornIE(InfoExtractor):
         age_limit = self._rta_search(webpage)
 
         # Get JSON parameters
-        json_params = self._search_regex(r'var currentVideo = new Video\((.*)\);', webpage, 'JSON parameters')
+        json_params = self._search_regex(
+            r'var currentVideo = new Video\((.*)\)[,;]',
+            webpage, 'JSON parameters')
         try:
             params = json.loads(json_params)
         except:

From 0df23ba9f9ddef005ec5f592ccb43b41564b5767 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 00:48:34 +0100
Subject: [PATCH 084/974] release 2014.12.06.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c568920d9..3ac525557 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.06'
+__version__ = '2014.12.06.1'

From 71155991218dc8c392e8e3431ab71a0f43687349 Mon Sep 17 00:00:00 2001
From: cryptonaut <cryptonaut64@gmail.com>
Date: Fri, 5 Dec 2014 21:49:41 -0800
Subject: [PATCH 085/974] [adultswim] Updated to work with new site format
 (fixes #4317)

---
 youtube_dl/extractor/adultswim.py | 206 +++++++++++++++++-------------
 1 file changed, 114 insertions(+), 92 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 0d05cbb4b..e4c78a81c 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -2,123 +2,147 @@
 from __future__ import unicode_literals
 
 import re
+import json
 
 from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+)
 
 
 class AdultSwimIE(InfoExtractor):
-    _VALID_URL = r'https?://video\.adultswim\.com/(?P<path>.+?)(?:\.html)?(?:\?.*)?(?:#.*)?$'
-    _TEST = {
-        'url': 'http://video.adultswim.com/rick-and-morty/close-rick-counters-of-the-rick-kind.html?x=y#title',
+    _VALID_URL = r'https?://(?:www\.)?adultswim\.com/videos/(?P<is_playlist>playlists/)?(?P<show_path>[^/]+)/(?P<episode_path>[^/?#]+)/?'
+
+    _TESTS = [{
+        'url': 'http://adultswim.com/videos/rick-and-morty/pilot',
+        'playlist': [
+           {
+                'md5': '247572debc75c7652f253c8daa51a14d',
+                'info_dict': {
+                    'id': 'rQxZvXQ4ROaSOqq-or2Mow-0',
+                    'ext': 'flv',
+                    'title': 'Rick and Morty - Pilot Part 1',
+                    'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
+                },
+            },
+            {
+                'md5': '77b0e037a4b20ec6b98671c4c379f48d',
+                'info_dict': {
+                    'id': 'rQxZvXQ4ROaSOqq-or2Mow-3',
+                    'ext': 'flv',
+                    'title': 'Rick and Morty - Pilot Part 4',
+                    'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
+                },
+            },
+        ],
+        'info_dict': {
+            'title': 'Rick and Morty - Pilot',
+            'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
+        }
+    }, {
+        'url': 'http://www.adultswim.com/videos/playlists/american-parenting/putting-francine-out-of-business/',
         'playlist': [
             {
-                'md5': '4da359ec73b58df4575cd01a610ba5dc',
+                'md5': '2eb5c06d0f9a1539da3718d897f13ec5',
                 'info_dict': {
-                    'id': '8a250ba1450996e901453d7f02ca02f5',
+                    'id': '-t8CamQlQ2aYZ49ItZCFog-0',
                     'ext': 'flv',
-                    'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 1',
-                    'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
-                    'uploader': 'Rick and Morty',
-                    'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
-                }
-            },
-            {
-                'md5': 'ffbdf55af9331c509d95350bd0cc1819',
-                'info_dict': {
-                    'id': '8a250ba1450996e901453d7f4bd102f6',
-                    'ext': 'flv',
-                    'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 2',
-                    'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
-                    'uploader': 'Rick and Morty',
-                    'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
-                }
-            },
-            {
-                'md5': 'b92409635540304280b4b6c36bd14a0a',
-                'info_dict': {
-                    'id': '8a250ba1450996e901453d7fa73c02f7',
-                    'ext': 'flv',
-                    'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 3',
-                    'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
-                    'uploader': 'Rick and Morty',
-                    'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
-                }
-            },
-            {
-                'md5': 'e8818891d60e47b29cd89d7b0278156d',
-                'info_dict': {
-                    'id': '8a250ba1450996e901453d7fc8ba02f8',
-                    'ext': 'flv',
-                    'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 4',
-                    'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
-                    'uploader': 'Rick and Morty',
-                    'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
-                }
+                    'title': 'American Dad - Putting Francine Out of Business',
+                    'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
+                },
             }
-        ]
-    }
+        ],
+        'info_dict': {
+            'title': 'American Dad - Putting Francine Out of Business',
+            'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
+        },
+    }]
 
-    _video_extensions = {
-        '3500': 'flv',
-        '640': 'mp4',
-        '150': 'mp4',
-        'ipad': 'm3u8',
-        'iphone': 'm3u8'
-    }
-    _video_dimensions = {
-        '3500': (1280, 720),
-        '640': (480, 270),
-        '150': (320, 180)
-    }
+    @staticmethod
+    def find_video_info(collection, slug):
+        for video in collection.get('videos'):
+            if video.get('slug') == slug:
+                return video
+
+    @staticmethod
+    def find_collection_by_linkURL(collections, linkURL):
+        for collection in collections:
+            if collection.get('linkURL') == linkURL:
+                return collection
+
+    @staticmethod
+    def find_collection_containing_video(collections, slug):
+        for collection in collections:
+            for video in collection.get('videos'):
+                if video.get('slug') == slug:
+                    return collection, video
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_path = mobj.group('path')
+        show_path = mobj.group('show_path')
+        episode_path = mobj.group('episode_path')
+        is_playlist = True if mobj.group('is_playlist') else False
 
-        webpage = self._download_webpage(url, video_path)
-        episode_id = self._html_search_regex(
-            r'<link rel="video_src" href="http://i\.adultswim\.com/adultswim/adultswimtv/tools/swf/viralplayer.swf\?id=([0-9a-f]+?)"\s*/?\s*>',
-            webpage, 'episode_id')
-        title = self._og_search_title(webpage)
+        webpage = self._download_webpage(url, episode_path)
 
-        index_url = 'http://asfix.adultswim.com/asfix-svc/episodeSearch/getEpisodesByIDs?networkName=AS&ids=%s' % episode_id
-        idoc = self._download_xml(index_url, title, 'Downloading episode index', 'Unable to download episode index')
+        # Extract the value of `bootstrappedData` from the Javascript in the page.
+        bootstrappedDataJS = self._search_regex(r'var bootstrappedData = ({.*});', webpage, episode_path)
 
-        episode_el = idoc.find('.//episode')
-        show_title = episode_el.attrib.get('collectionTitle')
-        episode_title = episode_el.attrib.get('title')
-        thumbnail = episode_el.attrib.get('thumbnailUrl')
-        description = episode_el.find('./description').text.strip()
+        try:
+            bootstrappedData = json.loads(bootstrappedDataJS)
+        except ValueError as ve:
+            errmsg = '%s: Failed to parse JSON ' % episode_path
+            raise ExtractorError(errmsg, cause=ve)
+
+        # Downloading videos from a /videos/playlist/ URL needs to be handled differently.
+        # NOTE: We are only downloading one video (the current one) not the playlist
+        if is_playlist:
+            collections = bootstrappedData['playlists']['collections']
+            collection = self.find_collection_by_linkURL(collections, show_path)
+            video_info = self.find_video_info(collection, episode_path)
+
+            show_title = video_info['showTitle']
+            segment_ids = [video_info['videoPlaybackID']]
+        else:
+            collections = bootstrappedData['show']['collections']
+            collection, video_info = self.find_collection_containing_video(collections, episode_path)
+
+            show = bootstrappedData['show']
+            show_title = show['title']
+            segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
+
+        episode_id = video_info['id']
+        episode_title = video_info['title']
+        episode_description = video_info['description']
+        episode_duration = video_info.get('duration')
 
         entries = []
-        segment_els = episode_el.findall('./segments/segment')
+        for part_num, segment_id in enumerate(segment_ids):
+            segment_url = 'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=mobile' % segment_id
 
-        for part_num, segment_el in enumerate(segment_els):
-            segment_id = segment_el.attrib.get('id')
-            segment_title = '%s %s part %d' % (show_title, episode_title, part_num + 1)
-            thumbnail = segment_el.attrib.get('thumbnailUrl')
-            duration = segment_el.attrib.get('duration')
+            segment_title = '%s - %s' % (show_title, episode_title)
+            if len(segment_ids) > 1:
+                segment_title += ' Part %d' % (part_num + 1)
 
-            segment_url = 'http://asfix.adultswim.com/asfix-svc/episodeservices/getCvpPlaylist?networkName=AS&id=%s' % segment_id
             idoc = self._download_xml(
                 segment_url, segment_title,
                 'Downloading segment information', 'Unable to download segment information')
 
+            segment_duration = idoc.find('.//trt').text.strip()
+
             formats = []
             file_els = idoc.findall('.//files/file')
 
             for file_el in file_els:
                 bitrate = file_el.attrib.get('bitrate')
-                type = file_el.attrib.get('type')
-                width, height = self._video_dimensions.get(bitrate, (None, None))
+                ftype = file_el.attrib.get('type')
+
                 formats.append({
-                    'format_id': '%s-%s' % (bitrate, type),
-                    'url': file_el.text,
-                    'ext': self._video_extensions.get(bitrate, 'mp4'),
+                    'format_id': '%s_%s' % (bitrate, ftype),
+                    'url': file_el.text.strip(),
                     # The bitrate may not be a number (for example: 'iphone')
                     'tbr': int(bitrate) if bitrate.isdigit() else None,
-                    'height': height,
-                    'width': width
+                    'quality': 1 if ftype == 'hd' else -1
                 })
 
             self._sort_formats(formats)
@@ -127,18 +151,16 @@ class AdultSwimIE(InfoExtractor):
                 'id': segment_id,
                 'title': segment_title,
                 'formats': formats,
-                'uploader': show_title,
-                'thumbnail': thumbnail,
-                'duration': duration,
-                'description': description
+                'duration': segment_duration,
+                'description': episode_description
             })
 
         return {
             '_type': 'playlist',
             'id': episode_id,
-            'display_id': video_path,
+            'display_id': episode_path,
             'entries': entries,
-            'title': '%s %s' % (show_title, episode_title),
-            'description': description,
-            'thumbnail': thumbnail
+            'title': '%s - %s' % (show_title, episode_title),
+            'description': episode_description,
+            'duration': episode_duration
         }

From 754f0008ec6b87316ce3e4807f150726ff2af3c5 Mon Sep 17 00:00:00 2001
From: netanel <netanelmaman0@gmail.com>
Date: Sat, 6 Dec 2014 09:20:35 +0200
Subject: [PATCH 086/974] fix increment operator

---
 youtube_dl/extractor/eighttracks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index 101a09699..f093592a8 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -135,7 +135,7 @@ class EightTracksIE(InfoExtractor):
                     if download_tries > 3:
                         raise
                     else:
-                        ++download_tries
+                        download_tries += 1
                         time.sleep(avg_song_duration)
 
             api_data = json.loads(api_json)

From 1a9f8b1ad4f56c42e28f45bb727d0fc20f71c7a7 Mon Sep 17 00:00:00 2001
From: cryptonaut <cryptonaut64@gmail.com>
Date: Sat, 6 Dec 2014 01:42:58 -0800
Subject: [PATCH 087/974] [nba] Improve _VALID_URL regex (fixes #4387)

Allows for optional trailing / or /index.html
---
 youtube_dl/extractor/nba.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index f69fe0925..842ea66a1 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class NBAIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)(?:/index\.html)?(?:\?.*)?$'
+    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
     _TEST = {
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
         'md5': 'c0edcfc37607344e2ff8f13c378c88a4',

From 0b5cc1983ecc7f407092d5431807341d82610aa8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 11:15:25 +0100
Subject: [PATCH 088/974] [nba] Modernize

---
 youtube_dl/extractor/nba.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 842ea66a1..cd021332b 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -24,9 +24,7 @@ class NBAIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_url = 'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4'
@@ -37,7 +35,7 @@ class NBAIE(InfoExtractor):
 
         description = self._og_search_description(webpage)
         duration = parse_duration(
-            self._html_search_meta('duration', webpage, 'duration', fatal=False))
+            self._html_search_meta('duration', webpage, 'duration'))
 
         return {
             'id': shortened_video_id,

From 6a3e0103bb84ca65e11e8473ff41bf7796d998a2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 11:26:17 +0100
Subject: [PATCH 089/974] [nba] Add test for #4387

---
 youtube_dl/extractor/nba.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index cd021332b..2b3a3daa7 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class NBAIE(InfoExtractor):
     _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
         'md5': 'c0edcfc37607344e2ff8f13c378c88a4',
         'info_dict': {
@@ -21,7 +21,10 @@ class NBAIE(InfoExtractor):
             'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
             'duration': 181,
         },
-    }
+    }, {
+        'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 5f9b83944da7c531f82ace0ce9275c2661e7187e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 12:14:26 +0100
Subject: [PATCH 090/974] [ffmpeg] Improve version check and call it from hls
 (Fixes #4377)

---
 test/test_utils.py                 | 6 ++++++
 youtube_dl/downloader/hls.py       | 4 ++++
 youtube_dl/postprocessor/ffmpeg.py | 6 +++---
 youtube_dl/utils.py                | 2 +-
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index badab1370..5c7bf5175 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -48,6 +48,7 @@ from youtube_dl.utils import (
     intlist_to_bytes,
     args_to_str,
     parse_filesize,
+    version_tuple,
 )
 
 
@@ -381,5 +382,10 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_filesize('1.2Tb'), 1200000000000)
         self.assertEqual(parse_filesize('1,24 KB'), 1240)
 
+    def test_version_tuple(self):
+        self.assertEqual(version_tuple('1'), (1,))
+        self.assertEqual(version_tuple('10.23.344'), (10, 23, 344))
+        self.assertEqual(version_tuple('10-6'), (10, 6))  # avconv style
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 954beffd5..ad26cfa40 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -4,6 +4,7 @@ import os
 import re
 import subprocess
 
+from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from .common import FileDownloader
 from ..utils import (
     compat_urlparse,
@@ -32,6 +33,9 @@ class HlsFD(FileDownloader):
             return False
         cmd = [program] + args
 
+        ffpp = FFmpegPostProcessor(downloader=self)
+        ffpp.check_version()
+
         retval = subprocess.call(cmd)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 9303b8378..965ded4c1 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -37,11 +37,11 @@ class FFmpegPostProcessor(PostProcessor):
         if not self._executable:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
-        REQUIRED_VERSION = '1.0'
+        required_version = '10-0' if self._uses_avconv() else '1.0'
         if is_outdated_version(
-                self._versions[self._executable], REQUIRED_VERSION):
+                self._versions[self._executable], required_version):
             warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
-                self._executable, self._executable, REQUIRED_VERSION)
+                self._executable, self._executable, required_version)
             if self._downloader:
                 self._downloader.report_warning(warning)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5efb4c59a..2e70cc791 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1502,7 +1502,7 @@ def limit_length(s, length):
 
 
 def version_tuple(v):
-    return [int(e) for e in v.split('.')]
+    return tuple(int(e) for e in re.split(r'[-.]', v))
 
 
 def is_outdated_version(version, limit, assume_new=True):

From 9ff6772790975ecf0d06c5511bfa7f1dee0955c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 12:20:54 +0100
Subject: [PATCH 091/974] [youtube] Modernize

---
 youtube_dl/extractor/youtube.py | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f452a90d8..addef9594 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1229,7 +1229,7 @@ class YoutubeTopListIE(YoutubePlaylistIE):
 
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = 'YouTube.com channels'
-    _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
+    _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
     _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
     _MORE_PAGES_URL = 'https://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
     IE_NAME = 'youtube:channel'
@@ -1247,13 +1247,8 @@ class YoutubeChannelIE(InfoExtractor):
         return ids_in_page
 
     def _real_extract(self, url):
-        # Extract channel id
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
+        channel_id = self._match_id(url)
 
-        # Download channel page
-        channel_id = mobj.group(1)
         video_ids = []
         url = 'https://www.youtube.com/channel/%s/videos' % channel_id
         channel_page = self._download_webpage(url, channel_id)
@@ -1290,7 +1285,7 @@ class YoutubeChannelIE(InfoExtractor):
 
 class YoutubeUserIE(InfoExtractor):
     IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
-    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
+    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
     _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/users/%s'
     _GDATA_PAGE_SIZE = 50
     _GDATA_URL = 'https://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
@@ -1318,12 +1313,7 @@ class YoutubeUserIE(InfoExtractor):
             return super(YoutubeUserIE, cls).suitable(url)
 
     def _real_extract(self, url):
-        # Extract username
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
-
-        username = mobj.group(1)
+        username = self._match_id(url)
 
         # Download video ids using YouTube Data API. Result size per
         # query is limited (currently to 50 videos) so we need to query

From db175341c7d80a9f78ddd78d8e93b1286f725a0f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 12:32:01 +0100
Subject: [PATCH 092/974] Credit @cryptonaut for adultswim (#4388)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 4d3b9ddc1..0b2ef8540 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -90,3 +90,4 @@ Matthew Rayfield
 t0mm0
 Tithen-Firion
 Zack Fernandes
+cryptonaut

From 47d7c64274f9cde33daab5f54b3707ffb19d2f20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 12:36:23 +0100
Subject: [PATCH 093/974] [test_utils] Make test more realistically (#4377)

---
 test/test_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 5c7bf5175..aaa293ff8 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -385,7 +385,7 @@ class TestUtil(unittest.TestCase):
     def test_version_tuple(self):
         self.assertEqual(version_tuple('1'), (1,))
         self.assertEqual(version_tuple('10.23.344'), (10, 23, 344))
-        self.assertEqual(version_tuple('10-6'), (10, 6))  # avconv style
+        self.assertEqual(version_tuple('10.1-6'), (10, 1, 6))  # avconv style
 
 if __name__ == '__main__':
     unittest.main()

From 6591fdf51f194d4d140fb3da582361766a8ec01c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 13:59:10 +0100
Subject: [PATCH 094/974] [tagesschau] Look at the right place for download
 links

---
 youtube_dl/extractor/tagesschau.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index aed670941..bfe07b024 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -70,7 +70,7 @@ class TagesschauIE(InfoExtractor):
                 webpage, 'download links')
             links = re.finditer(
                 r'<div class="button" title="(?P<title>[^"]*)"><a href="(?P<url>[^"]+)">(?P<name>.+?)</a></div>',
-                webpage)
+                download_text)
             formats = []
             for l in links:
                 format_id = self._search_regex(

From 7d70cf4157306e4b14c4f3967fad25911299acdf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 13:59:37 +0100
Subject: [PATCH 095/974] [nba] Remove unused import

---
 youtube_dl/extractor/nba.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 2b3a3daa7..862b706bf 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     remove_end,

From 158f8cadc0c29e9d43e1440a672a3f72f0e454c7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 14:01:59 +0100
Subject: [PATCH 096/974] [adultswim] PEP8

---
 youtube_dl/extractor/adultswim.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index e4c78a81c..39e4ca296 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -16,7 +16,7 @@ class AdultSwimIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://adultswim.com/videos/rick-and-morty/pilot',
         'playlist': [
-           {
+            {
                 'md5': '247572debc75c7652f253c8daa51a14d',
                 'info_dict': {
                     'id': 'rQxZvXQ4ROaSOqq-or2Mow-0',

From b82f815f373818ba99ee43660e9255e8f4ecac62 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 14:02:19 +0100
Subject: [PATCH 097/974] Allow iterators for playlist result entries

---
 youtube_dl/YoutubeDL.py         | 20 ++++++++++++++------
 youtube_dl/extractor/common.py  |  4 ++--
 youtube_dl/extractor/youtube.py | 18 ++++++++++--------
 3 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f89ac4e1d..56dc3d461 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -7,6 +7,7 @@ import collections
 import datetime
 import errno
 import io
+import itertools
 import json
 import locale
 import os
@@ -654,21 +655,28 @@ class YoutubeDL(object):
             if playlistend == -1:
                 playlistend = None
 
-            if isinstance(ie_result['entries'], list):
-                n_all_entries = len(ie_result['entries'])
-                entries = ie_result['entries'][playliststart:playlistend]
+            ie_entries = ie_result['entries']
+            if isinstance(ie_entries, list):
+                n_all_entries = len(ie_entries)
+                entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
-            else:
-                assert isinstance(ie_result['entries'], PagedList)
-                entries = ie_result['entries'].getslice(
+            elif isinstance(ie_entries, PagedList):
+                entries = ie_entries.getslice(
                     playliststart, playlistend)
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Downloading %d videos" %
                     (ie_result['extractor'], playlist, n_entries))
+            else:  # iterable
+                entries = list(itertools.islice(
+                    ie_entries, playliststart, playlistend))
+                n_entries = len(entries)
+                self.to_screen(
+                    "[%s] playlist %s: Downloading %d videos" %
+                    (ie_result['extractor'], playlist, n_entries))
 
             for i, entry in enumerate(entries, 1):
                 self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries))
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5b5e33cea..cb6081dd0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -158,8 +158,8 @@ class InfoExtractor(object):
 
 
     _type "playlist" indicates multiple videos.
-    There must be a key "entries", which is a list or a PagedList object, each
-    element of which is a valid dictionary under this specfication.
+    There must be a key "entries", which is a list, an iterable, or a PagedList
+    object, each element of which is a valid dictionary by this specification.
 
     Additionally, playlists can have "title" and "id" attributes with the same
     semantics as videos (see above).
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index addef9594..8b6e591a4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1262,8 +1262,12 @@ class YoutubeChannelIE(InfoExtractor):
             # The videos are contained in a single page
             # the ajax pages can't be used, they are empty
             video_ids = self.extract_videos_from_page(channel_page)
-        else:
-            # Download all channel pages using the json-based channel_ajax query
+            entries = [
+                self.url_result(video_id, 'Youtube', video_id=video_id)
+                for video_id in video_ids]
+            return self.playlist_result(entries, channel_id)
+
+        def _entries():
             for pagenum in itertools.count(1):
                 url = self._MORE_PAGES_URL % (pagenum, channel_id)
                 page = self._download_json(
@@ -1271,16 +1275,14 @@ class YoutubeChannelIE(InfoExtractor):
                     transform_source=uppercase_escape)
 
                 ids_in_page = self.extract_videos_from_page(page['content_html'])
-                video_ids.extend(ids_in_page)
+                for video_id in ids_in_page:
+                    yield self.url_result(
+                        video_id, 'Youtube', video_id=video_id)
 
                 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
                     break
 
-        self._downloader.to_screen('[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
-
-        url_entries = [self.url_result(video_id, 'Youtube', video_id=video_id)
-                       for video_id in video_ids]
-        return self.playlist_result(url_entries, channel_id)
+        return self.playlist_result(_entries(), channel_id)
 
 
 class YoutubeUserIE(InfoExtractor):

From 6ac4e8065a506f92db919ff063cf4d2f00bc81f0 Mon Sep 17 00:00:00 2001
From: Grom PE <i@grompe.org.ru>
Date: Sat, 6 Dec 2014 20:15:41 +0700
Subject: [PATCH 098/974] Fix utils.py for PyPy on Windows

The line
```python
from __future__ import unicode_literals
```
introduced in commit [ecc0c5ee01f0e5bdd6af0c32cb5b4adcb2a2f78c](https://github.com/rg3/youtube-dl/commit/ecc0c5ee01f0e5bdd6af0c32cb5b4adcb2a2f78c) broke youtube-dl for PyPy on Windows, making it unable to locate WinAPI functions.
Error: "TypeError: function name must be a string or integer"

Adding "b" prefix to strings with WinAPI function names fixes it.
---
 youtube_dl/utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2e70cc791..2b0f4e589 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -808,22 +808,22 @@ def _windows_write_string(s, out):
 
     GetStdHandle = ctypes.WINFUNCTYPE(
         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
-        ("GetStdHandle", ctypes.windll.kernel32))
+        (b"GetStdHandle", ctypes.windll.kernel32))
     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
 
     WriteConsoleW = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
-        ctypes.wintypes.LPVOID)(("WriteConsoleW", ctypes.windll.kernel32))
+        ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
     written = ctypes.wintypes.DWORD(0)
 
-    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(("GetFileType", ctypes.windll.kernel32))
+    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
     FILE_TYPE_CHAR = 0x0002
     FILE_TYPE_REMOTE = 0x8000
     GetConsoleMode = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
         ctypes.POINTER(ctypes.wintypes.DWORD))(
-        ("GetConsoleMode", ctypes.windll.kernel32))
+        (b"GetConsoleMode", ctypes.windll.kernel32))
     INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
 
     def not_a_console(handle):

From 04d02a9d57c834d3044871ce7dcc90932e686e46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Dec 2014 21:24:20 +0600
Subject: [PATCH 099/974] [twitch] Add login support (#3986)

---
 youtube_dl/extractor/twitch.py | 42 ++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 36aa1ad6e..397d167e8 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import itertools
@@ -5,6 +6,8 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    compat_urllib_parse,
+    compat_urllib_request,
     ExtractorError,
     parse_iso8601,
 )
@@ -24,6 +27,7 @@ class TwitchIE(InfoExtractor):
         """
     _PAGE_LIMIT = 100
     _API_BASE = 'https://api.twitch.tv'
+    _LOGIN_URL = 'https://secure.twitch.tv/user/login'
     _TESTS = [{
         'url': 'http://www.twitch.tv/riotgames/b/577357806',
         'info_dict': {
@@ -109,6 +113,44 @@ class TwitchIE(InfoExtractor):
             'view_count': info['views'],
         }
 
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, 'Downloading login page')
+
+        authenticity_token = self._search_regex(
+            r'<input name="authenticity_token" type="hidden" value="([^"]+)"',
+            login_page, 'authenticity token')
+
+        login_form = {
+            'utf8': '✓'.encode('utf-8'),
+            'authenticity_token': authenticity_token,
+            'redirect_on_login': '',
+            'embed_form': 'false',
+            'mp_source_action': '',
+            'follow': '',
+            'user[login]': username,
+            'user[password]': password,
+        }
+
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        request.add_header('Referer', self._LOGIN_URL)
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        m = re.search(
+            r"id=([\"'])login_error_message\1[^>]*>(?P<msg>[^<]+)", response)
+        if m:
+            raise ExtractorError(
+                'Unable to login: %s' % m.group('msg').strip(), expected=True)
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj.group('chapterid'):

From 6d0886204a920e64606688b1217835d10e47d281 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Dec 2014 23:01:52 +0600
Subject: [PATCH 100/974] [radio.de] Add support for radio.de websites (Closes
 #4393)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/radiode.py  | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/radiode.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4c9c44b31..33a22d76e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -307,6 +307,7 @@ from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .pyvideo import PyvideoIE
 from .quickvid import QuickVidIE
+from .radiode import RadioDeIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
diff --git a/youtube_dl/extractor/radiode.py b/youtube_dl/extractor/radiode.py
new file mode 100644
index 000000000..f95bc9454
--- /dev/null
+++ b/youtube_dl/extractor/radiode.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+
+
+class RadioDeIE(InfoExtractor):
+    IE_NAME = 'radio.de'
+    _VALID_URL = r'https?://(?P<id>.+?)\.(?:radio\.(?:de|at|fr|pt|es|pl|it)|rad\.io)'
+    _TEST = {
+        'url': 'http://ndr2.radio.de/',
+        'md5': '3b4cdd011bc59174596b6145cda474a4',
+        'info_dict': {
+            'id': 'ndr2',
+            'ext': 'mp3',
+            'title': 're:^NDR 2 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:591c49c702db1a33751625ebfb67f273',
+            'thumbnail': 're:^https?://.*\.png',
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        radio_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, radio_id)
+
+        broadcast = json.loads(self._search_regex(
+            r'_getBroadcast\s*=\s*function\(\s*\)\s*{\s*return\s+({.+?})\s*;\s*}',
+            webpage, 'broadcast'))
+
+        title = self._live_title(broadcast['name'])
+        description = broadcast.get('description') or broadcast.get('shortDescription')
+        thumbnail = broadcast.get('picture4Url') or broadcast.get('picture4TransUrl')
+
+        formats = [{
+            'url': stream['streamUrl'],
+            'ext': stream['streamContentFormat'].lower(),
+            'acodec': stream['streamContentFormat'],
+            'abr': stream['bitRate'],
+            'asr': stream['sampleRate']
+        } for stream in broadcast['streamUrls']]
+        self._sort_formats(formats)
+
+        return {
+            'id': radio_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'is_live': True,
+            'formats': formats,
+        }

From c84890f7082237b02a5fc61657b2ea513836a85e Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Sat, 6 Dec 2014 18:28:21 +0100
Subject: [PATCH 101/974] [prosiebensat1] Add support for playlists (fixes
 #4357)

---
 youtube_dl/extractor/prosiebensat1.py | 47 +++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 32d747ede..c6e539e5a 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -8,6 +8,7 @@ from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse,
     unified_strdate,
+    ExtractorError,
 )
 
 
@@ -152,6 +153,15 @@ class ProSiebenSat1IE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://www.prosieben.de/tv/joko-gegen-klaas/videos/playlists/episode-8-ganze-folge-playlist',
+            'info_dict': {
+                'id': '439664',
+                'title': 'Episode 8 - Ganze Folge - Playlist',
+                'description': 'Das finale und härteste Duell aller Zeiten ist vorbei! Der Weltmeister für dieses Jahr steht! Alle packenden Duelle der achten Episode von "Joko gegen Klaas - das Duell um die Welt" seht ihr hier noch einmal in voller Länge!',
+            },
+            'playlist_count': 2,
+        },
     ]
 
     _CLIPID_REGEXES = [
@@ -178,11 +188,48 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<span style="padding-left: 4px;line-height:20px; color:#404040">(\d{2}\.\d{2}\.\d{4})</span>',
         r'(\d{2}\.\d{2}\.\d{4}) \| \d{2}:\d{2} Min<br/>',
     ]
+    _ITEM_TYPE_REGEXES = [
+        r"'itemType'\s*:\s*'([^']*)'",
+    ]
+    _ITEM_ID_REGEXES = [
+        r"'itemId'\s*:\s*'([^']*)'",
+    ]
+    _PLAYLIST_CLIPS_REGEXES = [
+        r'data-qvt=.+?<a href="([^"]+)"',
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        item_type = self._html_search_regex(self._ITEM_TYPE_REGEXES, webpage, 'item type', default='CLIP')
+        if item_type == 'CLIP':
+            return self._clip_extract(url, webpage)
+        elif item_type == 'PLAYLIST':
+            playlist_id = self._html_search_regex(self._ITEM_ID_REGEXES, webpage, 'playlist id')
+
+            for regex in self._PLAYLIST_CLIPS_REGEXES:
+                playlist_clips = re.findall(regex, webpage, re.DOTALL)
+                if playlist_clips:
+                    title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
+                    description = self._html_search_regex(self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
+                    root_url = re.match('(.+?//.+?)/', url).group(1)
+
+                    return {
+                        '_type': 'playlist',
+                        'id': playlist_id,
+                        'title': title,
+                        'description': description,
+                        'entries': [self._clip_extract(root_url + clip_path) for clip_path in playlist_clips]
+                    }
+        else:
+            raise ExtractorError('Unknown item type "%s"' % item_type)
+
+    def _clip_extract(self, url, webpage=None):
+        if webpage is None:
+            video_id = self._match_id(url)
+            webpage = self._download_webpage(url, video_id)
+
         clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
 
         access_token = 'testclient'

From 8d1c8cae9ce9027058d90b246b9fa30ea88602ef Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Sat, 6 Dec 2014 19:21:05 +0100
Subject: [PATCH 102/974] [prosiebensat1] Fix broken tests

---
 youtube_dl/extractor/prosiebensat1.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index c6e539e5a..8c9451f40 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -86,7 +86,7 @@ class ProSiebenSat1IE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Im Interview: Kai Wiesinger',
                 'description': 'md5:e4e5370652ec63b95023e914190b4eb9',
-                'upload_date': '20140225',
+                'upload_date': '20140203',
                 'duration': 522.56,
             },
             'params': {
@@ -101,7 +101,7 @@ class ProSiebenSat1IE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Jagd auf Fertigkost im Elsthal - Teil 2',
                 'description': 'md5:2669cde3febe9bce13904f701e774eb6',
-                'upload_date': '20140225',
+                'upload_date': '20141014',
                 'duration': 2410.44,
             },
             'params': {
@@ -168,6 +168,7 @@ class ProSiebenSat1IE(InfoExtractor):
         r'"clip_id"\s*:\s+"(\d+)"',
         r'clipid: "(\d+)"',
         r'clip[iI]d=(\d+)',
+        r"'itemImageUrl'\s*:\s*'/dynamic/thumbnails/full/\d+/(\d+)",
     ]
     _TITLE_REGEXES = [
         r'<h2 class="subtitle" itemprop="name">\s*(.+?)</h2>',

From acf5cbfe93ec4f828fef75486fa8b4e587a64baa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Dec 2014 01:46:30 +0600
Subject: [PATCH 103/974] [extractor/common] Add description to playlist_result

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cb6081dd0..2faaf6226 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -439,7 +439,7 @@ class InfoExtractor(object):
         return video_info
 
     @staticmethod
-    def playlist_result(entries, playlist_id=None, playlist_title=None):
+    def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
         """Returns a playlist"""
         video_info = {'_type': 'playlist',
                       'entries': entries}
@@ -447,6 +447,8 @@ class InfoExtractor(object):
             video_info['id'] = playlist_id
         if playlist_title:
             video_info['title'] = playlist_title
+        if playlist_description:
+            video_info['description'] = playlist_description
         return video_info
 
     def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):

From 6a52eed80edbb0f1e65a74def32d84236ac6e640 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Dec 2014 01:46:44 +0600
Subject: [PATCH 104/974] [prosiebensat1] Improve and simplify

---
 youtube_dl/extractor/prosiebensat1.py | 74 +++++++++++++--------------
 1 file changed, 36 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 8c9451f40..1262793c8 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -8,7 +8,6 @@ from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse,
     unified_strdate,
-    ExtractorError,
 )
 
 
@@ -158,7 +157,7 @@ class ProSiebenSat1IE(InfoExtractor):
             'info_dict': {
                 'id': '439664',
                 'title': 'Episode 8 - Ganze Folge - Playlist',
-                'description': 'Das finale und härteste Duell aller Zeiten ist vorbei! Der Weltmeister für dieses Jahr steht! Alle packenden Duelle der achten Episode von "Joko gegen Klaas - das Duell um die Welt" seht ihr hier noch einmal in voller Länge!',
+                'description': 'md5:63b8963e71f481782aeea877658dec84',
             },
             'playlist_count': 2,
         },
@@ -189,48 +188,19 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<span style="padding-left: 4px;line-height:20px; color:#404040">(\d{2}\.\d{2}\.\d{4})</span>',
         r'(\d{2}\.\d{2}\.\d{4}) \| \d{2}:\d{2} Min<br/>',
     ]
-    _ITEM_TYPE_REGEXES = [
+    _PAGE_TYPE_REGEXES = [
+        r'<meta name="page_type" content="([^"]+)">',
         r"'itemType'\s*:\s*'([^']*)'",
     ]
-    _ITEM_ID_REGEXES = [
+    _PLAYLIST_ID_REGEXES = [
+        r'content[iI]d=(\d+)',
         r"'itemId'\s*:\s*'([^']*)'",
     ]
-    _PLAYLIST_CLIPS_REGEXES = [
-        r'data-qvt=.+?<a href="([^"]+)"',
+    _PLAYLIST_CLIP_REGEXES = [
+        r'(?s)data-qvt=.+?<a href="([^"]+)"',
     ]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        item_type = self._html_search_regex(self._ITEM_TYPE_REGEXES, webpage, 'item type', default='CLIP')
-        if item_type == 'CLIP':
-            return self._clip_extract(url, webpage)
-        elif item_type == 'PLAYLIST':
-            playlist_id = self._html_search_regex(self._ITEM_ID_REGEXES, webpage, 'playlist id')
-
-            for regex in self._PLAYLIST_CLIPS_REGEXES:
-                playlist_clips = re.findall(regex, webpage, re.DOTALL)
-                if playlist_clips:
-                    title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
-                    description = self._html_search_regex(self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
-                    root_url = re.match('(.+?//.+?)/', url).group(1)
-
-                    return {
-                        '_type': 'playlist',
-                        'id': playlist_id,
-                        'title': title,
-                        'description': description,
-                        'entries': [self._clip_extract(root_url + clip_path) for clip_path in playlist_clips]
-                    }
-        else:
-            raise ExtractorError('Unknown item type "%s"' % item_type)
-
-    def _clip_extract(self, url, webpage=None):
-        if webpage is None:
-            video_id = self._match_id(url)
-            webpage = self._download_webpage(url, video_id)
-
+    def _extract_clip(self, url, webpage):
         clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
 
         access_token = 'testclient'
@@ -329,3 +299,31 @@ class ProSiebenSat1IE(InfoExtractor):
             'duration': duration,
             'formats': formats,
         }
+
+    def _extract_playlist(self, url, webpage):
+        playlist_id = self._html_search_regex(
+            self._PLAYLIST_ID_REGEXES, webpage, 'playlist id')
+        for regex in self._PLAYLIST_CLIP_REGEXES:
+            playlist_clips = re.findall(regex, webpage)
+            if playlist_clips:
+                title = self._html_search_regex(
+                    self._TITLE_REGEXES, webpage, 'title')
+                description = self._html_search_regex(
+                    self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
+                entries = [
+                    self.url_result(
+                        re.match('(.+?//.+?)/', url).group(1) + clip_path,
+                        'ProSiebenSat1')
+                    for clip_path in playlist_clips]
+                return self.playlist_result(entries, playlist_id, title, description)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        page_type = self._search_regex(
+            self._PAGE_TYPE_REGEXES, webpage,
+            'page type', default='clip').lower()
+        if page_type == 'clip':
+            return self._extract_clip(url, webpage)
+        elif page_type == 'playlist':
+            return self._extract_playlist(url, webpage)

From 0ef4d4ab7e15833031cd43211a38464a9ab9aa17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Dec 2014 01:48:45 +0600
Subject: [PATCH 105/974] Credit @akretz for prosiebensat1 playlist support
 (#4394)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 0b2ef8540..bfa00f91b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -91,3 +91,4 @@ t0mm0
 Tithen-Firion
 Zack Fernandes
 cryptonaut
+Adrian Kretz

From 16ae61f65515229b5cb4f5a60936bed3fa1b2b23 Mon Sep 17 00:00:00 2001
From: cryptonaut <cryptonaut64@gmail.com>
Date: Sat, 6 Dec 2014 12:47:29 -0800
Subject: [PATCH 106/974] Handle --get-url with merged formats (fixes #2883)

Outputs one URL per line
---
 youtube_dl/YoutubeDL.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 56dc3d461..248367039 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -942,8 +942,12 @@ class YoutubeDL(object):
         if self.params.get('forceid', False):
             self.to_stdout(info_dict['id'])
         if self.params.get('forceurl', False):
-            # For RTMP URLs, also include the playpath
-            self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
+            if info_dict.get('requested_formats') is not None:
+                for f in info_dict['requested_formats']:
+                    self.to_stdout(f['url'] + f.get('play_path', ''))
+            else:
+                # For RTMP URLs, also include the playpath
+                self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
             self.to_stdout(info_dict['thumbnail'])
         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:

From 603c92080fd9923f4a17f9ee67fac99494a92bc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 7 Dec 2014 11:26:07 +0100
Subject: [PATCH 107/974] [nhl] Make sure we add '_sd' before the extension
 (fixes #4397)

'.replace' would find the first dot in the path.
---
 youtube_dl/extractor/nhl.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 0244368e9..b2f40344f 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import re
 import json
+import os
 
 from .common import InfoExtractor
 from ..compat import (
@@ -26,7 +27,8 @@ class NHLBaseInfoExtractor(InfoExtractor):
         initial_video_url = info['publishPoint']
         if info['formats'] == '1':
             parsed_url = compat_urllib_parse_urlparse(initial_video_url)
-            path = parsed_url.path.replace('.', '_sd.', 1)
+            filename, ext = os.path.splitext(parsed_url.path)
+            path = '%s_sd%s' % (filename, ext)
             data = compat_urllib_parse.urlencode({
                 'type': 'fvod',
                 'path': compat_urlparse.urlunparse(parsed_url[:2] + (path,) + parsed_url[3:])

From 684712076f1ade9dbe432ad95d5796ec278b73ce Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Sun, 7 Dec 2014 11:51:02 +0100
Subject: [PATCH 108/974] add direct screenwavemedia.com URL support

---
 youtube_dl/extractor/__init__.py              |  2 +-
 .../{cinemassacre.py => screenwavemedia.py}   | 76 +++++++++++++++----
 2 files changed, 61 insertions(+), 17 deletions(-)
 rename youtube_dl/extractor/{cinemassacre.py => screenwavemedia.py} (64%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 33a22d76e..c0dcdaf02 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -50,7 +50,6 @@ from .cbsnews import CBSNewsIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
-from .cinemassacre import CinemassacreIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .clipsyndicate import ClipsyndicateIE
@@ -335,6 +334,7 @@ from .savefrom import SaveFromIE
 from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
+from .screenwavemedia import ScreenwaveMediaIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
 from .sexykarma import SexyKarmaIE
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/screenwavemedia.py
similarity index 64%
rename from youtube_dl/extractor/cinemassacre.py
rename to youtube_dl/extractor/screenwavemedia.py
index b7fa73c3b..3fb3bfa32 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -9,15 +9,18 @@ from ..utils import (
     int_or_none,
 )
 
+class ScreenwaveMediaIE(InfoExtractor):
+    _VALID_URL = r'(?:http://)?(?' \
+        r':(?P<generic>player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<video_id>.+))' \
+        r'|(?P<cinemassacre>(?:www\.)?cinemassacre\.com/(?P<cm_date_Y>[0-9]{4})/(?P<cm_date_m>[0-9]{2})/(?P<cm_date_d>[0-9]{2})/(?P<cm_display_id>[^?#/]+))' \
+        r')'
 
-class CinemassacreIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?cinemassacre\.com/(?P<date_Y>[0-9]{4})/(?P<date_m>[0-9]{2})/(?P<date_d>[0-9]{2})/(?P<display_id>[^?#/]+)'
     _TESTS = [
         {
             'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
             'md5': 'fde81fbafaee331785f58cd6c0d46190',
             'info_dict': {
-                'id': '19911',
+                'id': 'Cinemasssacre-19911',
                 'ext': 'mp4',
                 'upload_date': '20121110',
                 'title': '“Angry Video Game Nerd: The Movie” – Trailer',
@@ -28,7 +31,7 @@ class CinemassacreIE(InfoExtractor):
             'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
             'md5': 'd72f10cd39eac4215048f62ab477a511',
             'info_dict': {
-                'id': '521be8ef82b16',
+                'id': 'Cinemasssacre-521be8ef82b16',
                 'ext': 'mp4',
                 'upload_date': '20131002',
                 'title': 'The Mummy’s Hand (1940)',
@@ -36,18 +39,16 @@ class CinemassacreIE(InfoExtractor):
         }
     ]
 
-    def _real_extract(self, url):
+    def _cinemassacre_get_info(self, url):
         mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('cm_display_id')
 
         webpage = self._download_webpage(url, display_id)
-        video_date = mobj.group('date_Y') + mobj.group('date_m') + mobj.group('date_d')
-        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<full_video_id>(?:Cinemassacre-)?(?P<video_id>.+?)))"', webpage)
+        video_date = mobj.group('cm_date_Y') + mobj.group('cm_date_m') + mobj.group('cm_date_d')
+        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"', webpage)
         if not mobj:
             raise ExtractorError('Can\'t extract embed url and video id')
         playerdata_url = mobj.group('embed_url')
-        video_id = mobj.group('video_id')
-        full_video_id = mobj.group('full_video_id')
 
         video_title = self._html_search_regex(
             r'<title>(?P<title>.+?)\|', webpage, 'title')
@@ -56,10 +57,28 @@ class CinemassacreIE(InfoExtractor):
             webpage, 'description', flags=re.DOTALL, fatal=False)
         video_thumbnail = self._og_search_thumbnail(webpage)
 
-        playerdata = self._download_webpage(playerdata_url, video_id, 'Downloading player webpage')
+        return {
+            'title': video_title,
+            'description': video_description,
+            'upload_date': video_date,
+            'thumbnail': video_thumbnail,
+            '_embed_url': playerdata_url,
+        }
 
+    def _screenwavemedia_get_info(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        if not mobj:
+            raise ExtractorError('Can\'t extract embed url and video id')
+        video_id = mobj.group('video_id')
+
+        playerdata = self._download_webpage(url, video_id, 'Downloading player webpage')
+
+        vidtitle = self._search_regex(
+            r'\'vidtitle\'\s*:\s*"([^\']+)"', playerdata, 'vidtitle').replace('\\/', '/')
         vidurl = self._search_regex(
             r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/')
+        pageurl = self._search_regex(
+            r'\'pageurl\'\s*:\s*"([^\']+)"', playerdata, 'pageurl', fatal=False).replace('\\/', '/')
 
         videolist_url = None
 
@@ -67,7 +86,7 @@ class CinemassacreIE(InfoExtractor):
         if mobj:
             videoserver = mobj.group('videoserver')
             mobj = re.search(r'\'vidid\'\s*:\s*"(?P<vidid>[^\']+)"', playerdata)
-            vidid = mobj.group('vidid') if mobj else full_video_id
+            vidid = mobj.group('vidid') if mobj else video_id
             videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
         else:
             mobj = re.search(r"file\s*:\s*'(?P<smil>http.+?/jwplayer\.smil)'", playerdata)
@@ -110,9 +129,34 @@ class CinemassacreIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': video_title,
+            'title': vidtitle,
             'formats': formats,
-            'description': video_description,
-            'upload_date': video_date,
-            'thumbnail': video_thumbnail,
+            '_episode_page': pageurl,
         }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        swm_info = None
+        site_info = None
+
+        if mobj.group('generic'):
+            swm_info = self._screenwavemedia_get_info(url)
+            url = swm_info['_episode_page']
+            mobj = re.match(self._VALID_URL, url)
+
+        if mobj:
+            if mobj.group('cinemassacre'):
+                site_info = self._cinemassacre_get_info(url)
+
+        if not swm_info:
+            if site_info:
+                swm_info = self._screenwavemedia_get_info(site_info['_embed_url'])
+
+        if not swm_info:
+            raise ExtractorError("Failed to extract metadata for this URL")
+
+        if site_info:
+            swm_info.update(site_info)
+
+        return swm_info

From ce36339575da408c5d795f691d07d2e310292b28 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Sun, 7 Dec 2014 11:52:03 +0100
Subject: [PATCH 109/974] add teamfourstar.com support

---
 youtube_dl/extractor/screenwavemedia.py | 32 +++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 3fb3bfa32..cf9b8e0de 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    month_by_name,
     int_or_none,
 )
 
@@ -13,6 +14,7 @@ class ScreenwaveMediaIE(InfoExtractor):
     _VALID_URL = r'(?:http://)?(?' \
         r':(?P<generic>player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<video_id>.+))' \
         r'|(?P<cinemassacre>(?:www\.)?cinemassacre\.com/(?P<cm_date_Y>[0-9]{4})/(?P<cm_date_m>[0-9]{2})/(?P<cm_date_d>[0-9]{2})/(?P<cm_display_id>[^?#/]+))' \
+        r'|(?P<teamfourstar>(?:www\.)?teamfourstar\.com/video/(?P<tfs_display_id>[a-z0-9\-]+)/?)' \
         r')'
 
     _TESTS = [
@@ -65,6 +67,34 @@ class ScreenwaveMediaIE(InfoExtractor):
             '_embed_url': playerdata_url,
         }
 
+    def _teamfourstar_get_info(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('tfs_display_id')
+        webpage = self._download_webpage(url, display_id)
+
+        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"', webpage)
+        if not mobj:
+            raise ExtractorError('Can\'t extract embed url and video id')
+        playerdata_url = mobj.group('embed_url')
+
+        video_title = self._html_search_regex(
+            r'<div class="heroheadingtitle">(?P<title>.+?)</div>', webpage, 'title')
+        video_date = self._html_search_regex(
+            r'<div class="heroheadingdate">(?P<date>.+?)</div>', webpage, 'date')
+        mobj = re.match('(?P<month>[A-Z][a-z]+) (?P<day>\d+), (?P<year>\d+)', video_date)
+        video_date = '%04u%02u%02u' % (int(mobj.group('year')), month_by_name(mobj.group('month')), int(mobj.group('day')))
+        video_description = self._html_search_regex(
+            r'<div class="postcontent">(?P<description>.+?)</div>', webpage, 'description', flags=re.DOTALL)
+        video_thumbnail = self._og_search_thumbnail(webpage)
+
+        return {
+            'title': video_title,
+            'description': video_description,
+            'upload_date': video_date,
+            'thumbnail': video_thumbnail,
+            '_embed_url': playerdata_url,
+        }
+
     def _screenwavemedia_get_info(self, url):
         mobj = re.match(self._VALID_URL, url)
         if not mobj:
@@ -148,6 +178,8 @@ class ScreenwaveMediaIE(InfoExtractor):
         if mobj:
             if mobj.group('cinemassacre'):
                 site_info = self._cinemassacre_get_info(url)
+            elif mobj.group('teamfourstar'):
+                site_info = self._teamfourstar_get_info(url)
 
         if not swm_info:
             if site_info:

From 819039ee63fc74f15d8c2138791d025a74743590 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Dec 2014 22:03:02 +0600
Subject: [PATCH 110/974] [tvigle] Update test and modernize

---
 youtube_dl/extractor/tvigle.py | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index d81d1d1a6..ba65996dc 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -1,32 +1,30 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
-    str_to_int,
+    parse_age_limit,
 )
 
 
 class TvigleIE(InfoExtractor):
     IE_NAME = 'tvigle'
     IE_DESC = 'Интернет-телевидение Tvigle.ru'
-    _VALID_URL = r'http://(?:www\.)?tvigle\.ru/(?:[^/]+/)+(?P<display_id>[^/]+)/$'
+    _VALID_URL = r'http://(?:www\.)?tvigle\.ru/(?:[^/]+/)+(?P<id>[^/]+)/$'
 
     _TESTS = [
         {
-            'url': 'http://www.tvigle.ru/video/brat/',
-            'md5': 'ff4344a4894b0524441fb6f8218dc716',
+            'url': 'http://www.tvigle.ru/video/sokrat/',
+            'md5': '36514aed3657d4f70b4b2cef8eb520cd',
             'info_dict': {
-                'id': '5118490',
-                'display_id': 'brat',
-                'ext': 'mp4',
-                'title': 'Брат',
-                'description': 'md5:d16ac7c0b47052ea51fddb92c4e413eb',
-                'duration': 5722.6,
-                'age_limit': 16,
+                'id': '1848932',
+                'display_id': 'sokrat',
+                'ext': 'flv',
+                'title': 'Сократ',
+                'description': 'md5:a05bd01be310074d5833efc6743be95e',
+                'duration': 6586,
+                'age_limit': 0,
             },
         },
         {
@@ -44,8 +42,7 @@ class TvigleIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
 
@@ -60,8 +57,8 @@ class TvigleIE(InfoExtractor):
         title = item['title']
         description = item['description']
         thumbnail = item['thumbnail']
-        duration = float_or_none(item['durationMilliseconds'], 1000)
-        age_limit = str_to_int(item['ageRestrictions'])
+        duration = float_or_none(item.get('durationMilliseconds'), 1000)
+        age_limit = parse_age_limit(item.get('ageRestrictions'))
 
         formats = []
         for vcodec, fmts in item['videos'].items():

From 09a5dd2d3be02a5fca7639a77c5dcf970bfa0e8f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 8 Dec 2014 17:17:15 +0100
Subject: [PATCH 111/974] [bliptv] Add support for audio-only files (Fixes
 #4404)

---
 youtube_dl/extractor/bliptv.py | 42 ++++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index da47f27bd..14b814120 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -4,13 +4,17 @@ import re
 
 from .common import InfoExtractor
 from .subtitles import SubtitlesInfoExtractor
-from ..utils import (
-    compat_urllib_request,
-    unescapeHTML,
-    parse_iso8601,
-    compat_urlparse,
-    clean_html,
+
+from ..compat import (
     compat_str,
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
+    clean_html,
+    int_or_none,
+    parse_iso8601,
+    unescapeHTML,
 )
 
 
@@ -78,7 +82,25 @@ class BlipTVIE(SubtitlesInfoExtractor):
                 'uploader': 'NostalgiaCritic',
                 'uploader_id': '246467',
             }
-        }
+        },
+        {
+            # https://github.com/rg3/youtube-dl/pull/4404
+            'note': 'Audio only',
+            'url': 'http://blip.tv/hilarios-productions/weekly-manga-recap-kingdom-7119982',
+            'md5': '76c0a56f24e769ceaab21fbb6416a351',
+            'info_dict': {
+                'id': '7103299',
+                'ext': 'flv',
+                'title': 'Weekly Manga Recap: Kingdom',
+                'description': 'And then Shin breaks the enemy line, and he&apos;s all like HWAH! And then he slices a guy and it&apos;s all like FWASHING! And... it&apos;s really hard to describe the best parts of this series without breaking down into sound effects, okay?',
+                'timestamp': 1417660321,
+                'upload_date': '20141204',
+                'uploader': 'The Rollo T',
+                'uploader_id': '407429',
+                'duration': 7251,
+                'vcodec': 'none',
+            }
+        },
     ]
 
     def _real_extract(self, url):
@@ -145,11 +167,11 @@ class BlipTVIE(SubtitlesInfoExtractor):
                     'url': real_url,
                     'format_id': role,
                     'format_note': media_type,
-                    'vcodec': media_content.get(blip('vcodec')),
+                    'vcodec': media_content.get(blip('vcodec')) or 'none',
                     'acodec': media_content.get(blip('acodec')),
                     'filesize': media_content.get('filesize'),
-                    'width': int(media_content.get('width')),
-                    'height': int(media_content.get('height')),
+                    'width': int_or_none(media_content.get('width')),
+                    'height': int_or_none(media_content.get('height')),
                 })
         self._sort_formats(formats)
 

From f58766ce5cc5a1c6d651560591bad119787e94b2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 9 Dec 2014 10:58:06 +0100
Subject: [PATCH 112/974] [extractor/common] Document ie_key in url results

---
 youtube_dl/extractor/common.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2faaf6226..cfd25cd21 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -174,9 +174,10 @@ class InfoExtractor(object):
     _type "url" indicates that the video must be extracted from another
     location, possibly by a different extractor. Its only required key is:
     "url" - the next URL to extract.
-
-    Additionally, it may have properties believed to be identical to the
-    resolved entity, for example "title" if the title of the referred video is
+    The key "ie_key" can be set to the class name (minus the trailing "IE",
+    e.g. "Youtube") if the extractor class is known in advance.
+    Additionally, the dictionary may have any properties of the resolved entity
+    known in advance, for example "title" if the title of the referred video is
     known ahead of time.
 
 

From e89a2aabedf24c145698a96a192a28a55a16fe1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Dec 2014 22:28:28 +0600
Subject: [PATCH 113/974] [extractor/common] Add generic SMIL formats
 extraction routine

---
 youtube_dl/extractor/common.py | 43 ++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cfd25cd21..2277ec6ab 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -793,6 +793,49 @@ class InfoExtractor(object):
         self._sort_formats(formats)
         return formats
 
+    # TODO: improve extraction
+    def _extract_smil_formats(self, smil_url, video_id):
+        smil = self._download_xml(
+            smil_url, video_id, 'Downloading SMIL file',
+            'Unable to download SMIL file')
+
+        base = smil.find('./head/meta').get('base')
+
+        formats = []
+        rtmp_count = 0
+        for video in smil.findall('./body/switch/video'):
+            src = video.get('src')
+            if not src:
+                continue
+            bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+            width = int_or_none(video.get('width'))
+            height = int_or_none(video.get('height'))
+            proto = video.get('proto')
+            if not proto:
+                if base:
+                    if base.startswith('rtmp'):
+                        proto = 'rtmp'
+                    elif base.startswith('http'):
+                        proto = 'http'
+            ext = video.get('ext')
+            if proto == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(src, video_id, ext))
+            elif proto == 'rtmp':
+                rtmp_count += 1
+                streamer = video.get('streamer') or base
+                formats.append({
+                    'url': streamer,
+                    'play_path': src,
+                    'ext': 'flv',
+                    'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
+                    'tbr': bitrate,
+                    'width': width,
+                    'height': height,
+                })
+        self._sort_formats(formats)
+
+        return formats
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()

From b89a9386874630c6bfa5cecc59af200ac3206813 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Dec 2014 22:29:01 +0600
Subject: [PATCH 114/974] [bet] Add extractor (Closes #4416)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/bet.py      | 108 +++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 youtube_dl/extractor/bet.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 33a22d76e..9f2dc803b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -30,6 +30,7 @@ from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
+from .bet import BetIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
 from .blinkx import BlinkxIE
diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
new file mode 100644
index 000000000..c1fc433f7
--- /dev/null
+++ b/youtube_dl/extractor/bet.py
@@ -0,0 +1,108 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urllib_parse,
+    xpath_text,
+    xpath_with_ns,
+    int_or_none,
+    parse_iso8601,
+)
+
+
+class BetIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html'
+    _TESTS = [
+        {
+            'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
+            'info_dict': {
+                'id': '417cd61c-c793-4e8e-b006-e445ecc45add',
+                'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
+                'ext': 'flv',
+                'title': 'BET News Presents: A Conversation With President Obama',
+                'description': 'md5:5a88d8ae912c1b33e090290af7ec33c6',
+                'duration': 1534,
+                'timestamp': 1418075340,
+                'upload_date': '20141208',
+                'uploader': 'admin',
+                'thumbnail': 're:(?i)^https?://.*\.jpg$',
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
+            'info_dict': {
+                'id': '4160e53b-ad41-43b1-980f-8d85f63121f4',
+                'display_id': 'justice-for-ferguson-a-community-reacts',
+                'ext': 'flv',
+                'title': 'Justice for Ferguson: A Community Reacts',
+                'description': 'A BET News special.',
+                'duration': 1696,
+                'timestamp': 1416942360,
+                'upload_date': '20141125',
+                'uploader': 'admin',
+                'thumbnail': 're:(?i)^https?://.*\.jpg$',
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        }
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        media_url = compat_urllib_parse.unquote(self._search_regex(
+            [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
+            webpage, 'media URL'))
+
+        mrss = self._download_xml(media_url, display_id)
+
+        item = mrss.find('./channel/item')
+
+        NS_MAP = {
+            'dc': 'http://purl.org/dc/elements/1.1/',
+            'media': 'http://search.yahoo.com/mrss/',
+            'ka': 'http://kickapps.com/karss',
+        }
+
+        title = xpath_text(item, './title', 'title')
+        description = xpath_text(
+            item, './description', 'description', fatal=False)
+
+        video_id = xpath_text(item, './guid', 'video id', fatal=False)
+
+        timestamp = parse_iso8601(xpath_text(
+            item, xpath_with_ns('./dc:date', NS_MAP),
+            'upload date', fatal=False))
+        uploader = xpath_text(
+            item, xpath_with_ns('./dc:creator', NS_MAP),
+            'uploader', fatal=False)
+
+        media_content = item.find(
+            xpath_with_ns('./media:content', NS_MAP))
+        duration = int_or_none(media_content.get('duration'))
+        smil_url = media_content.get('url')
+
+        thumbnail = media_content.find(
+            xpath_with_ns('./media:thumbnail', NS_MAP)).get('url')
+
+        formats = self._extract_smil_formats(smil_url, display_id)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'duration': duration,
+            'formats': formats,
+        }

From 83e865a37051ceeb0024490dce6849b8fd6a6943 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 9 Dec 2014 23:11:26 +0100
Subject: [PATCH 115/974] Fix PEP8 issue E713

---
 youtube_dl/compat.py                          | 2 +-
 youtube_dl/postprocessor/execafterdownload.py | 2 +-
 youtube_dl/update.py                          | 2 +-
 youtube_dl/utils.py                           | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 27596687d..f4a85443e 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -247,7 +247,7 @@ else:
                 userhome = compat_getenv('HOME')
             elif 'USERPROFILE' in os.environ:
                 userhome = compat_getenv('USERPROFILE')
-            elif not 'HOMEPATH' in os.environ:
+            elif 'HOMEPATH' not in os.environ:
                 return path
             else:
                 try:
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 09db43611..75c0f7bbe 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -14,7 +14,7 @@ class ExecAfterDownloadPP(PostProcessor):
 
     def run(self, information):
         cmd = self.exec_cmd
-        if not '{}' in cmd:
+        if '{}' not in cmd:
             cmd += ' {}'
 
         cmd = cmd.replace('{}', shlex_quote(information['filepath']))
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 4c07a558e..2d2703368 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -79,7 +79,7 @@ def update_self(to_screen, verbose):
             to_screen(compat_str(traceback.format_exc()))
         to_screen('ERROR: can\'t obtain versions info. Please try again later.')
         return
-    if not 'signature' in versions_info:
+    if 'signature' not in versions_info:
         to_screen('ERROR: the versions file is not signed or corrupted. Aborting.')
         return
     signature = versions_info['signature']
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2e70cc791..a95d2c942 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1024,7 +1024,7 @@ def smuggle_url(url, data):
 
 
 def unsmuggle_url(smug_url, default=None):
-    if not '#__youtubedl_smuggle' in smug_url:
+    if '#__youtubedl_smuggle' not in smug_url:
         return smug_url, default
     url, _, sdata = smug_url.rpartition('#')
     jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]

From f20b52778b32023b5b56245c36f5e638d7879515 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 12:21:40 +0100
Subject: [PATCH 116/974] release 2014.12.10

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3ac525557..21b7758bf 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.06.1'
+__version__ = '2014.12.10'

From 774e208f94574ef9c62e82ac5ad0c57f245a8752 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 13:21:24 +0100
Subject: [PATCH 117/974] [youtube] Handle missing DASH manifest (Fixes #4421,
 fixes #4420)

---
 youtube_dl/extractor/youtube.py | 113 +++++++++++++++++++-------------
 1 file changed, 68 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8b6e591a4..329690742 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -417,6 +417,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'upload_date': '20140605',
             },
         },
+        # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
+        {
+            'url': '__2ABJjxzNo',
+            'info_dict': {
+                'id': '__2ABJjxzNo',
+                'ext': 'mp4',
+                'upload_date': '20100430',
+                'uploader_id': 'deadmau5',
+                'description': 'md5:12c56784b8032162bb936a5f76d55360',
+                'uploader': 'deadmau5',
+                'title': 'Deadmau5 - Some Chords (HD)',
+            },
+            'expected_warnings': [
+                'DASH manifest missing',
+            ]
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -666,6 +682,45 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
+    def _parse_dash_manifest(self, video_id, dash_manifest_url):
+        def decrypt_sig(mobj):
+            s = mobj.group(1)
+            dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
+            return '/signature/%s' % dec_s
+        dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url)
+        dash_doc = self._download_xml(
+            dash_manifest_url, video_id,
+            note='Downloading DASH manifest',
+            errnote='Could not download DASH manifest')
+
+        formats = []
+        for r in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
+            url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
+            if url_el is None:
+                continue
+            format_id = r.attrib['id']
+            video_url = url_el.text
+            filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
+            f = {
+                'format_id': format_id,
+                'url': video_url,
+                'width': int_or_none(r.attrib.get('width')),
+                'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
+                'asr': int_or_none(r.attrib.get('audioSamplingRate')),
+                'filesize': filesize,
+                'fps': int_or_none(r.attrib.get('frameRate')),
+            }
+            try:
+                existing_format = next(
+                    fo for fo in formats
+                    if fo['format_id'] == format_id)
+            except StopIteration:
+                f.update(self._formats.get(format_id, {}))
+                formats.append(f)
+            else:
+                existing_format.update(f)
+        return formats
+
     def _real_extract(self, url):
         proto = (
             'http' if self._downloader.params.get('prefer_insecure', False)
@@ -943,51 +998,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
-            try:
-                # The DASH manifest used needs to be the one from the original video_webpage.
-                # The one found in get_video_info seems to be using different signatures.
-                # However, in the case of an age restriction there won't be any embedded dashmpd in the video_webpage.
-                # Luckily, it seems, this case uses some kind of default signature (len == 86), so the
-                # combination of get_video_info and the _static_decrypt_signature() decryption fallback will work here.
-                dash_manifest_url = video_info.get('dashmpd')[0]
-
-                def decrypt_sig(mobj):
-                    s = mobj.group(1)
-                    dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
-                    return '/signature/%s' % dec_s
-                dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url)
-                dash_doc = self._download_xml(
-                    dash_manifest_url, video_id,
-                    note='Downloading DASH manifest',
-                    errnote='Could not download DASH manifest')
-                for r in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
-                    url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
-                    if url_el is None:
-                        continue
-                    format_id = r.attrib['id']
-                    video_url = url_el.text
-                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
-                    f = {
-                        'format_id': format_id,
-                        'url': video_url,
-                        'width': int_or_none(r.attrib.get('width')),
-                        'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
-                        'asr': int_or_none(r.attrib.get('audioSamplingRate')),
-                        'filesize': filesize,
-                        'fps': int_or_none(r.attrib.get('frameRate')),
-                    }
-                    try:
-                        existing_format = next(
-                            fo for fo in formats
-                            if fo['format_id'] == format_id)
-                    except StopIteration:
-                        f.update(self._formats.get(format_id, {}))
-                        formats.append(f)
-                    else:
-                        existing_format.update(f)
-
-            except (ExtractorError, KeyError) as e:
-                self.report_warning('Skipping DASH manifest: %r' % e, video_id)
+            dash_mpd = video_info.get('dashmpd')
+            if not dash_mpd:
+                self.report_warning('%s: DASH manifest missing' % video_id)
+            else:
+                dash_manifest_url = dash_mpd[0]
+                try:
+                    dash_formats = self._parse_dash_manifest(
+                        video_id, dash_manifest_url)
+                except (ExtractorError, KeyError) as e:
+                    self.report_warning(
+                        'Skipping DASH manifest: %r' % e, video_id)
+                else:
+                    formats.extend(dash_formats)
 
         self._sort_formats(formats)
 

From b8a6114309971ff4fcfd7b617fcf0321a3ecfc1b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 13:21:49 +0100
Subject: [PATCH 118/974] release 2014.12.10.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 21b7758bf..bbd779764 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.10'
+__version__ = '2014.12.10.1'

From da276600148ce17714a93ea9f1de8fd38736f468 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 14:39:00 +0100
Subject: [PATCH 119/974] [youtube] Pass in all variables to DASH manifest
 (Fixes #4424)

---
 youtube_dl/extractor/youtube.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 329690742..42441d9fa 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -682,7 +682,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
-    def _parse_dash_manifest(self, video_id, dash_manifest_url):
+    def _parse_dash_manifest(
+            self, video_id, dash_manifest_url, player_url, age_gate):
         def decrypt_sig(mobj):
             s = mobj.group(1)
             dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
@@ -1005,7 +1006,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 dash_manifest_url = dash_mpd[0]
                 try:
                     dash_formats = self._parse_dash_manifest(
-                        video_id, dash_manifest_url)
+                        video_id, dash_manifest_url, player_url, age_gate)
                 except (ExtractorError, KeyError) as e:
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)

From 5f3544baa36b76b31b030e07ef6c120b1dd432c4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 14:39:06 +0100
Subject: [PATCH 120/974] release 2014.12.10.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index bbd779764..4ca1f095a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.10.1'
+__version__ = '2014.12.10.2'

From 2fc9f2b41d362f15c7d61fd2b16c541e53d03b87 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 15:18:34 +0100
Subject: [PATCH 121/974] [facebook] Make thumbnail and duration optional

Fixes #4425.
Looks like both properties aren't given to us anymore. For now, just fall back to not returning them.
---
 youtube_dl/extractor/facebook.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 2139f68aa..1ad4e77a8 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -13,9 +13,10 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
-    urlencode_postdata,
     ExtractorError,
+    int_or_none,
     limit_length,
+    urlencode_postdata,
 )
 
 
@@ -36,7 +37,6 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '637842556329505',
             'ext': 'mp4',
-            'duration': 38,
             'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
         }
     }, {
@@ -107,9 +107,7 @@ class FacebookIE(InfoExtractor):
         self._login()
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         url = 'https://www.facebook.com/video/video.php?v=%s' % video_id
         webpage = self._download_webpage(url, video_id)
 
@@ -149,6 +147,6 @@ class FacebookIE(InfoExtractor):
             'id': video_id,
             'title': video_title,
             'url': video_url,
-            'duration': int(video_data['video_duration']),
-            'thumbnail': video_data['thumbnail_src'],
+            'duration': int_or_none(video_data.get('video_duration')),
+            'thumbnail': video_data.get('thumbnail_src'),
         }

From 72e3ffeb743712c51f476196c1b22f4de23253f5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 15:19:08 +0100
Subject: [PATCH 122/974] release 2014.12.10.3

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4ca1f095a..0345b7eb6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.10.2'
+__version__ = '2014.12.10.3'

From a81bbebf44b6f405fcb3959b3fc2d2dc03b3bb59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Dec 2014 20:22:49 +0600
Subject: [PATCH 123/974] [smotri:broadcast] Fix extraction

---
 youtube_dl/extractor/smotri.py | 35 +++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 0751efc61..646af3cc9 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -274,15 +274,18 @@ class SmotriBroadcastIE(InfoExtractor):
         broadcast_page = self._download_webpage(broadcast_url, broadcast_id, 'Downloading broadcast page')
 
         if re.search('>Режиссер с логином <br/>"%s"<br/> <span>не существует<' % broadcast_id, broadcast_page) is not None:
-            raise ExtractorError('Broadcast %s does not exist' % broadcast_id, expected=True)
+            raise ExtractorError(
+                'Broadcast %s does not exist' % broadcast_id, expected=True)
 
         # Adult content
         if re.search('EroConfirmText">', broadcast_page) is not None:
 
             (username, password) = self._get_login_info()
             if username is None:
-                raise ExtractorError('Erotic broadcasts allowed only for registered users, '
-                                     'use --username and --password options to provide account credentials.', expected=True)
+                raise ExtractorError(
+                    'Erotic broadcasts allowed only for registered users, '
+                    'use --username and --password options to provide account credentials.',
+                    expected=True)
 
             login_form = {
                 'login-hint53': '1',
@@ -291,9 +294,11 @@ class SmotriBroadcastIE(InfoExtractor):
                 'password': password,
             }
 
-            request = compat_urllib_request.Request(broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
+            request = compat_urllib_request.Request(
+                broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
             request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-            broadcast_page = self._download_webpage(request, broadcast_id, 'Logging in and confirming age')
+            broadcast_page = self._download_webpage(
+                request, broadcast_id, 'Logging in and confirming age')
 
             if re.search('>Неверный логин или пароль<', broadcast_page) is not None:
                 raise ExtractorError('Unable to log in: bad username or password', expected=True)
@@ -303,7 +308,7 @@ class SmotriBroadcastIE(InfoExtractor):
             adult_content = False
 
         ticket = self._html_search_regex(
-            'window\.broadcast_control\.addFlashVar\\(\'file\', \'([^\']+)\'\\);',
+            r"window\.broadcast_control\.addFlashVar\('file'\s*,\s*'([^']+)'\)",
             broadcast_page, 'broadcast ticket')
 
         url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket
@@ -312,26 +317,31 @@ class SmotriBroadcastIE(InfoExtractor):
         if broadcast_password:
             url += '&pass=%s' % hashlib.md5(broadcast_password.encode('utf-8')).hexdigest()
 
-        broadcast_json_page = self._download_webpage(url, broadcast_id, 'Downloading broadcast JSON')
+        broadcast_json_page = self._download_webpage(
+            url, broadcast_id, 'Downloading broadcast JSON')
 
         try:
             broadcast_json = json.loads(broadcast_json_page)
 
             protected_broadcast = broadcast_json['_pass_protected'] == 1
             if protected_broadcast and not broadcast_password:
-                raise ExtractorError('This broadcast is protected by a password, use the --video-password option', expected=True)
+                raise ExtractorError(
+                    'This broadcast is protected by a password, use the --video-password option',
+                    expected=True)
 
             broadcast_offline = broadcast_json['is_play'] == 0
             if broadcast_offline:
                 raise ExtractorError('Broadcast %s is offline' % broadcast_id, expected=True)
 
             rtmp_url = broadcast_json['_server']
-            if not rtmp_url.startswith('rtmp://'):
+            mobj = re.search(r'^rtmp://[^/]+/(?P<app>.+)/?$', rtmp_url)
+            if not mobj:
                 raise ExtractorError('Unexpected broadcast rtmp URL')
 
             broadcast_playpath = broadcast_json['_streamName']
+            broadcast_app = '%s/%s' % (mobj.group('app'), broadcast_json['_vidURL'])
             broadcast_thumbnail = broadcast_json['_imgURL']
-            broadcast_title = broadcast_json['title']
+            broadcast_title = self._live_title(broadcast_json['title'])
             broadcast_description = broadcast_json['description']
             broadcaster_nick = broadcast_json['nick']
             broadcaster_login = broadcast_json['login']
@@ -352,6 +362,9 @@ class SmotriBroadcastIE(InfoExtractor):
             'age_limit': 18 if adult_content else 0,
             'ext': 'flv',
             'play_path': broadcast_playpath,
+            'player_url': 'http://pics.smotri.com/broadcast_play.swf',
+            'app': broadcast_app,
             'rtmp_live': True,
-            'rtmp_conn': rtmp_conn
+            'rtmp_conn': rtmp_conn,
+            'is_live': True,
         }

From 8560c618422214745caa4766292afa9c0d3f35c2 Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Wed, 10 Dec 2014 17:29:03 +0100
Subject: [PATCH 124/974] [zdf] Add support for channels

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/zdf.py      | 85 ++++++++++++++++++++++++++++++--
 2 files changed, 83 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9f2dc803b..027fb06f9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -526,7 +526,7 @@ from .youtube import (
     YoutubeUserIE,
     YoutubeWatchLaterIE,
 )
-from .zdf import ZDFIE
+from .zdf import ZDFIE, ZDFChannelIE
 from .zingmp3 import (
     ZingMp3SongIE,
     ZingMp3AlbumIE,
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 9ff00e26c..221f16686 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -86,6 +86,26 @@ def extract_from_xml_url(ie, video_id, xml_url):
     }
 
 
+def extract_channel_from_xml_url(ie, channel_id, xml_url):
+    doc = ie._download_xml(
+        xml_url, channel_id,
+        note='Downloading channel info',
+        errnote='Failed to download channel info')
+
+    title = doc.find('.//information/title').text
+    description = doc.find('.//information/detail').text
+    assets = [{'id': asset.find('./details/assetId').text,
+               'type': asset.find('./type').text,
+               } for asset in doc.findall('.//teasers/teaser')]
+
+    return {
+        'id': channel_id,
+        'title': title,
+        'description': description,
+        'assets': assets,
+    }
+
+
 class ZDFIE(InfoExtractor):
     _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*beitrag/(?:video/)?)(?P<id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'
 
@@ -104,8 +124,67 @@ class ZDFIE(InfoExtractor):
         'skip': 'Videos on ZDF.de are depublicised in short order',
     }
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
+    def _extract_video(self, video_id):
         xml_url = 'http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
         return extract_from_xml_url(self, video_id, xml_url)
+
+    def _real_extract(self, url):
+        return self._extract_video(self._match_id(url))
+
+
+class ZDFChannelIE(ZDFIE):
+    _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*kanaluebersicht/)(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://www.zdf.de/ZDFmediathek#/kanaluebersicht/1586442/sendung/Titanic',
+        'info_dict': {
+            'id': '1586442',
+            'title': 'Titanic',
+            'description': 'md5:444c048cfe3fdc2561be7de4bcbf1d04',
+        },
+        'playlist_count': 3,
+    }
+
+    def _extract_channel(self, channel_id):
+        def load_chunks(channel_id, chunk_length):
+            offset = 0
+            while True:
+                url = ('http://www.zdf.de/ZDFmediathek/xmlservice/web/aktuellste?ak=web&offset=%d&maxLength=%d&id=%s'
+                       % (offset, chunk_length, channel_id))
+                result = extract_channel_from_xml_url(self, channel_id, url)
+                yield result
+                if len(result['assets']) < chunk_length:
+                    return
+                offset += chunk_length
+
+        def load_channel(channel_id):
+            chunks = list(load_chunks(channel_id, 50))  # The server rejects higher values
+            assets = [asset for chunk in chunks for asset in chunk['assets']]
+            video_ids = [asset['id'] for asset in
+                         filter(lambda asset: asset['type'] == 'video',
+                                assets)]
+            topic_ids = [asset['id'] for asset in
+                         filter(lambda asset: asset['type'] == 'thema',
+                                assets)]
+            if topic_ids:
+                video_ids = reduce(list.__add__,
+                                   [load_channel(topic_id)['video_ids']
+                                    for topic_id in topic_ids],
+                                   video_ids)
+
+            result = chunks[0]
+            result['video_ids'] = video_ids
+            return result
+
+        channel = load_channel(channel_id)
+        return {
+            '_type': 'playlist',
+            'id': channel['id'],
+            'title': channel['title'],
+            'description': channel['description'],
+            'entries': [self._extract_video(video_id)
+                        for video_id in channel['video_ids']],
+        }
+
+    def _real_extract(self, url):
+        return self._extract_channel(self._match_id(url))

From 4bb4a18876f5489db77365528638da8d46890a38 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 10:08:17 +0100
Subject: [PATCH 125/974] [youtube] Fix imports

---
 youtube_dl/extractor/youtube.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 42441d9fa..6216028cf 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -14,23 +14,24 @@ from .common import InfoExtractor, SearchInfoExtractor
 from .subtitles import SubtitlesInfoExtractor
 from ..jsinterp import JSInterpreter
 from ..swfinterp import SWFInterpreter
-from ..utils import (
+from ..compat import (
     compat_chr,
     compat_parse_qs,
     compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
     compat_str,
-
+)
+from ..utils import (
     clean_html,
-    get_element_by_id,
-    get_element_by_attribute,
     ExtractorError,
+    get_element_by_attribute,
+    get_element_by_id,
     int_or_none,
     OnDemandPagedList,
+    orderedSet,
     unescapeHTML,
     unified_strdate,
-    orderedSet,
     uppercase_escape,
 )
 

From f8795e102b35198db550432c2eee43a1c2571093 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 10:29:30 +0100
Subject: [PATCH 126/974] [utils] Add "yesterday" as a date keyword

---
 youtube_dl/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a95d2c942..75f9594e6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -712,8 +712,10 @@ def date_from_str(date_str):
     Return a datetime object from a string in the format YYYYMMDD or
     (now|today)[+-][0-9](day|week|month|year)(s)?"""
     today = datetime.date.today()
-    if date_str == 'now'or date_str == 'today':
+    if date_str in ('now', 'today'):
         return today
+    if date_str == 'yesterday':
+        return today - datetime.timedelta(days=1)
     match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
     if match is not None:
         sign = match.group('sign')

From e0b9d47387327fb8d25d5bb2f2f4f80a3b77a4a8 Mon Sep 17 00:00:00 2001
From: 5moufl <malouito@gmail.com>
Date: Thu, 11 Dec 2014 14:25:26 +0100
Subject: [PATCH 127/974] [BehindKink] Update  URL extraction

---
 youtube_dl/extractor/behindkink.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/behindkink.py b/youtube_dl/extractor/behindkink.py
index 31fdc0dcc..84d8ef849 100644
--- a/youtube_dl/extractor/behindkink.py
+++ b/youtube_dl/extractor/behindkink.py
@@ -34,8 +34,7 @@ class BehindKinkIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         video_url = self._search_regex(
-            r"'file':\s*'([^']+)'",
-            webpage, 'URL base')
+            r'<source src="(.*?)" type="video/mp4" />', webpage, 'video URL')
 
         video_id = url_basename(video_url)
         video_id = video_id.split('_')[0]

From d6f78a354d77d32c9929ca99cb82d49aa7d7f8ab Mon Sep 17 00:00:00 2001
From: 5moufl <malouito@gmail.com>
Date: Thu, 11 Dec 2014 14:26:59 +0100
Subject: [PATCH 128/974] [BehindKink] Replace test

Old one is not accessible anymore
---
 youtube_dl/extractor/behindkink.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/behindkink.py b/youtube_dl/extractor/behindkink.py
index 84d8ef849..9e4f1c90f 100644
--- a/youtube_dl/extractor/behindkink.py
+++ b/youtube_dl/extractor/behindkink.py
@@ -10,15 +10,15 @@ from ..utils import url_basename
 class BehindKinkIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?behindkink\.com/(?P<year>[0-9]{4})/(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<id>[^/#?_]+)'
     _TEST = {
-        'url': 'http://www.behindkink.com/2014/08/14/ab1576-performers-voice-finally-heard-the-bill-is-killed/',
-        'md5': '41ad01222b8442089a55528fec43ec01',
+        'url': 'http://www.behindkink.com/2014/12/05/what-are-you-passionate-about-marley-blaze/',
+        'md5': '507b57d8fdcd75a41a9a7bdb7989c762',
         'info_dict': {
-            'id': '36370',
+            'id': '37127',
             'ext': 'mp4',
-            'title': 'AB1576 - PERFORMERS VOICE FINALLY HEARD - THE BILL IS KILLED!',
-            'description': 'The adult industry voice was finally heard as Assembly Bill 1576 remained\xa0 in suspense today at the Senate Appropriations Hearing. AB1576 was, among other industry damaging issues, a condom mandate...',
-            'upload_date': '20140814',
-            'thumbnail': 'http://www.behindkink.com/wp-content/uploads/2014/08/36370_AB1576_Win.jpg',
+            'title': 'What are you passionate about – Marley Blaze',
+            'description': 'Getting a better understanding of the talent that comes through the doors of the Armory is one of our missions at Behind Kink. Asking the question what are you passionate about helps us get a littl...',
+            'upload_date': '20141205',
+            'thumbnail': 'http://www.behindkink.com/wp-content/uploads/2014/12/blaze-1.jpg',
             'age_limit': 18,
         }
     }

From 09c82fbc9a6fc4c5dbb3b84031f31b39e1f9cee2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Dec 2014 20:06:19 +0600
Subject: [PATCH 129/974] [behindkink] Simplify

---
 youtube_dl/extractor/behindkink.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/behindkink.py b/youtube_dl/extractor/behindkink.py
index 9e4f1c90f..8c947526f 100644
--- a/youtube_dl/extractor/behindkink.py
+++ b/youtube_dl/extractor/behindkink.py
@@ -16,7 +16,7 @@ class BehindKinkIE(InfoExtractor):
             'id': '37127',
             'ext': 'mp4',
             'title': 'What are you passionate about – Marley Blaze',
-            'description': 'Getting a better understanding of the talent that comes through the doors of the Armory is one of our missions at Behind Kink. Asking the question what are you passionate about helps us get a littl...',
+            'description': 'md5:aee8e9611b4ff70186f752975d9b94b4',
             'upload_date': '20141205',
             'thumbnail': 'http://www.behindkink.com/wp-content/uploads/2014/12/blaze-1.jpg',
             'age_limit': 18,
@@ -26,25 +26,19 @@ class BehindKinkIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         display_id = mobj.group('id')
-        year = mobj.group('year')
-        month = mobj.group('month')
-        day = mobj.group('day')
-        upload_date = year + month + day
 
         webpage = self._download_webpage(url, display_id)
 
         video_url = self._search_regex(
-            r'<source src="(.*?)" type="video/mp4" />', webpage, 'video URL')
-
-        video_id = url_basename(video_url)
-        video_id = video_id.split('_')[0]
+            r'<source src="([^"]+)"', webpage, 'video URL')
+        video_id =  url_basename(video_url).split('_')[0]
+        upload_date = mobj.group('year') + mobj.group('month') + mobj.group('day')
 
         return {
             'id': video_id,
-            'url': video_url,
-            'ext': 'mp4',
-            'title': self._og_search_title(webpage),
             'display_id': display_id,
+            'url': video_url,
+            'title': self._og_search_title(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
             'upload_date': upload_date,

From bf671b605e013db8ecfa539bfe03689db830448b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Dec 2014 20:09:52 +0600
Subject: [PATCH 130/974] [behindkink] Remove superfluous whitespace

---
 youtube_dl/extractor/behindkink.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/behindkink.py b/youtube_dl/extractor/behindkink.py
index 8c947526f..1bdc25812 100644
--- a/youtube_dl/extractor/behindkink.py
+++ b/youtube_dl/extractor/behindkink.py
@@ -31,7 +31,7 @@ class BehindKinkIE(InfoExtractor):
 
         video_url = self._search_regex(
             r'<source src="([^"]+)"', webpage, 'video URL')
-        video_id =  url_basename(video_url).split('_')[0]
+        video_id = url_basename(video_url).split('_')[0]
         upload_date = mobj.group('year') + mobj.group('month') + mobj.group('day')
 
         return {

From 3c1b81b95735bf8fb065ba7a32f220c37b8a10dc Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 11 Dec 2014 16:58:45 +0200
Subject: [PATCH 131/974] [ntv] Rename `flash_ver` to `flash_version` in the
 format dict

RTMP downloader uses `flash_version`
---
 youtube_dl/extractor/ntv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ntv.py b/youtube_dl/extractor/ntv.py
index 13c8d79cd..ee740cd9c 100644
--- a/youtube_dl/extractor/ntv.py
+++ b/youtube_dl/extractor/ntv.py
@@ -130,7 +130,7 @@ class NTVIE(InfoExtractor):
                 'rtmp_conn': 'B:1',
                 'player_url': 'http://www.ntv.ru/swf/vps1.swf?update=20131128',
                 'page_url': 'http://www.ntv.ru',
-                'flash_ver': 'LNX 11,2,202,341',
+                'flash_version': 'LNX 11,2,202,341',
                 'rtmp_live': True,
                 'ext': 'flv',
                 'filesize': int(size.text),

From 24270b0301aacc307688d1da71ee0de6e24f3197 Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Wed, 10 Dec 2014 21:51:05 +0100
Subject: [PATCH 132/974] [youtube] The case that 'url_encoded_fmt_stream_map'
 or 'adaptive_fmts' is the empty string is handled accordingly (fixes #4431)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 42441d9fa..faac98ce0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -934,7 +934,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'url': video_info['conn'][0],
                 'player_url': player_url,
             }]
-        elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1:
+        elif len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1:
             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)

From 158731f83e2af39b3ecd8af3eaf4ce5b26ff6908 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 11 Dec 2014 17:07:50 +0200
Subject: [PATCH 133/974] [tvplay] Don't raise an exception if `is_geo_blocked`
 is True

Videos which return `is_geo_blocked' to be True can actually be downloaded from
the country to which the video is restricted
---
 youtube_dl/extractor/tvplay.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index eb9473754..0157392cc 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -182,8 +182,8 @@ class TVPlayIE(InfoExtractor):
             'http://playapi.mtgx.tv/v1/videos/%s' % video_id, video_id, 'Downloading video JSON')
 
         if video['is_geo_blocked']:
-            raise ExtractorError(
-                'This content is not available in your country due to copyright reasons', expected=True)
+            self.report_warning(
+                'This content might not be available in your country due to copyright reasons')
 
         streams = self._download_json(
             'http://playapi.mtgx.tv/v1/videos/stream/%s' % video_id, video_id, 'Downloading streams JSON')

From e52a40abf7519ba38a0607ea2069e39d684a12e2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 16:28:07 +0100
Subject: [PATCH 134/974] [youtube] Add test case for #4431

---
 youtube_dl/extractor/youtube.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6216028cf..4f6898c1a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -433,6 +433,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'expected_warnings': [
                 'DASH manifest missing',
             ]
+        },
+        # Olympics (https://github.com/rg3/youtube-dl/issues/4431)
+        {
+            'url': 'lqQg6PlCWgI',
+            'info_dict': {
+                'id': 'lqQg6PlCWgI',
+                'ext': 'mp4',
+                
+            }
         }
     ]
 

From 624dcebff617d3e995d1f0ddb27d604a881b2de2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 16:32:33 +0100
Subject: [PATCH 135/974] [youtube] Make category optional (#4442)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f8d5cdd5d..003aeaee6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -866,7 +866,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
         m_cat_container = self._search_regex(
             r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
-            video_webpage, 'categories', fatal=False)
+            video_webpage, 'categories', default=None)
         if m_cat_container:
             category = self._html_search_regex(
                 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',

From 75111274edeec5f7088730b7f9c5c623dae77f28 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 16:33:28 +0100
Subject: [PATCH 136/974] [youtube] Do not warn if DASH manifest is missing
 (#4442)

---
 youtube_dl/extractor/youtube.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 003aeaee6..f41086762 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1010,9 +1010,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
             dash_mpd = video_info.get('dashmpd')
-            if not dash_mpd:
-                self.report_warning('%s: DASH manifest missing' % video_id)
-            else:
+            if dash_mpd:
                 dash_manifest_url = dash_mpd[0]
                 try:
                     dash_formats = self._parse_dash_manifest(

From cbe2bd914d65fc717d31034234102cd25c7f7313 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 16:34:37 +0100
Subject: [PATCH 137/974] [youtube] Amend test

---
 youtube_dl/extractor/youtube.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f41086762..7b6179a2a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -440,9 +440,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'info_dict': {
                 'id': 'lqQg6PlCWgI',
                 'ext': 'mp4',
-                
+                'upload_date': '20120731',
+                'uploader_id': 'olympic',
+                'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
+                'uploader': 'Olympics',
+                'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
+            },
+            'params': {
+                'skip_download': 'requires avconv',
             }
-        }
+        },
     ]
 
     def __init__(self, *args, **kwargs):

From 9abd500a7402fe1f66e4177b61a7327b416b6b9d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 17:07:59 +0100
Subject: [PATCH 138/974] [zdf:channel] Simplify (#4427)

---
 youtube_dl/extractor/zdf.py | 111 +++++++++++++-----------------------
 1 file changed, 40 insertions(+), 71 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 221f16686..74c76a9a0 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -1,12 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import functools
 import re
 
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
+    OnDemandPagedList,
 )
 
 
@@ -86,28 +88,8 @@ def extract_from_xml_url(ie, video_id, xml_url):
     }
 
 
-def extract_channel_from_xml_url(ie, channel_id, xml_url):
-    doc = ie._download_xml(
-        xml_url, channel_id,
-        note='Downloading channel info',
-        errnote='Failed to download channel info')
-
-    title = doc.find('.//information/title').text
-    description = doc.find('.//information/detail').text
-    assets = [{'id': asset.find('./details/assetId').text,
-               'type': asset.find('./type').text,
-               } for asset in doc.findall('.//teasers/teaser')]
-
-    return {
-        'id': channel_id,
-        'title': title,
-        'description': description,
-        'assets': assets,
-    }
-
-
 class ZDFIE(InfoExtractor):
-    _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*beitrag/(?:video/)?)(?P<id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'
+    _VALID_URL = r'(?:zdf:|zdf:video:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/(.*beitrag/(?:video/)?))(?P<id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'
 
     _TEST = {
         'url': 'http://www.zdf.de/ZDFmediathek/beitrag/video/2037704/ZDFspezial---Ende-des-Machtpokers--?bc=sts;stt',
@@ -124,67 +106,54 @@ class ZDFIE(InfoExtractor):
         'skip': 'Videos on ZDF.de are depublicised in short order',
     }
 
-    def _extract_video(self, video_id):
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
         xml_url = 'http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
         return extract_from_xml_url(self, video_id, xml_url)
 
-    def _real_extract(self, url):
-        return self._extract_video(self._match_id(url))
-
-
-class ZDFChannelIE(ZDFIE):
-    _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*kanaluebersicht/)(?P<id>[0-9]+)'
 
+class ZDFChannelIE(InfoExtractor):
+    _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/)(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.zdf.de/ZDFmediathek#/kanaluebersicht/1586442/sendung/Titanic',
         'info_dict': {
             'id': '1586442',
-            'title': 'Titanic',
-            'description': 'md5:444c048cfe3fdc2561be7de4bcbf1d04',
         },
-        'playlist_count': 3,
+        'playlist_count': 4,
     }
+    _PAGE_SIZE = 50
 
-    def _extract_channel(self, channel_id):
-        def load_chunks(channel_id, chunk_length):
-            offset = 0
-            while True:
-                url = ('http://www.zdf.de/ZDFmediathek/xmlservice/web/aktuellste?ak=web&offset=%d&maxLength=%d&id=%s'
-                       % (offset, chunk_length, channel_id))
-                result = extract_channel_from_xml_url(self, channel_id, url)
-                yield result
-                if len(result['assets']) < chunk_length:
-                    return
-                offset += chunk_length
+    def _fetch_page(self, channel_id, page):
+        offset = page * self._PAGE_SIZE
+        xml_url = (
+            'http://www.zdf.de/ZDFmediathek/xmlservice/web/aktuellste?ak=web&offset=%d&maxLength=%d&id=%s'
+            % (offset, self._PAGE_SIZE, channel_id))
+        doc = self._download_xml(
+            xml_url, channel_id,
+            note='Downloading channel info',
+            errnote='Failed to download channel info')
 
-        def load_channel(channel_id):
-            chunks = list(load_chunks(channel_id, 50))  # The server rejects higher values
-            assets = [asset for chunk in chunks for asset in chunk['assets']]
-            video_ids = [asset['id'] for asset in
-                         filter(lambda asset: asset['type'] == 'video',
-                                assets)]
-            topic_ids = [asset['id'] for asset in
-                         filter(lambda asset: asset['type'] == 'thema',
-                                assets)]
-            if topic_ids:
-                video_ids = reduce(list.__add__,
-                                   [load_channel(topic_id)['video_ids']
-                                    for topic_id in topic_ids],
-                                   video_ids)
-
-            result = chunks[0]
-            result['video_ids'] = video_ids
-            return result
-
-        channel = load_channel(channel_id)
-        return {
-            '_type': 'playlist',
-            'id': channel['id'],
-            'title': channel['title'],
-            'description': channel['description'],
-            'entries': [self._extract_video(video_id)
-                        for video_id in channel['video_ids']],
-        }
+        title = doc.find('.//information/title').text
+        description = doc.find('.//information/detail').text
+        for asset in doc.findall('.//teasers/teaser'):
+            a_type = asset.find('./type').text
+            a_id = asset.find('./details/assetId').text
+            if a_type not in ('video', 'topic'):
+                continue
+            yield {
+                '_type': 'url',
+                'playlist_title': title,
+                'playlist_description': description,
+                'url': 'zdf:%s:%s' % (a_type, a_id),
+            }
 
     def _real_extract(self, url):
-        return self._extract_channel(self._match_id(url))
+        channel_id = self._match_id(url)
+        entries = OnDemandPagedList(
+            functools.partial(self._fetch_page, channel_id), self._PAGE_SIZE)
+
+        return {
+            '_type': 'playlist',
+            'id': channel_id,
+            'entries': entries,
+        }

From cb007f47c1685e1d17a8fc7be7920293610290a6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 17:08:31 +0100
Subject: [PATCH 139/974] release 2014.12.11

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0345b7eb6..b2a285bb2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.10.3'
+__version__ = '2014.12.11'

From 9c1aa1d66819ebce6e6cce42da3065ec91720620 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 11 Dec 2014 23:16:40 +0100
Subject: [PATCH 140/974] [mixcloud] Fix metadata extraction (fixes #4443)

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index bb8937c4d..55cc33a3e 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -70,7 +70,7 @@ class MixcloudIE(InfoExtractor):
             raise ExtractorError('Unable to extract track url')
 
         PREFIX = (
-            r'<div class="cloudcast-play-button-container[^"]*?"'
+            r'<span class="play-button[^"]*?"'
             r'(?:\s+[a-zA-Z0-9-]+(?:="[^"]+")?)*?\s+')
         title = self._html_search_regex(
             PREFIX + r'm-title="([^"]+)"', webpage, 'title')

From 807962f4a14c08d397dd7a015aaa3f8948f6af82 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 23:50:25 +0100
Subject: [PATCH 141/974] [pornhd] Adapt to new sources scheme (Fixes #4446)

---
 youtube_dl/extractor/pornhd.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index bac484c67..63472b33e 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -45,13 +45,18 @@ class PornHdIE(InfoExtractor):
         thumbnail = self._search_regex(
             r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
 
-        quality = qualities(['SD', 'HD'])
-        formats = [{
-            'url': source['file'],
-            'format_id': '%s-%s' % (source['label'], determine_ext(source['file'])),
-            'quality': quality(source['label']),
-        } for source in json.loads(js_to_json(self._search_regex(
-            r"(?s)'sources'\s*:\s*(\[.+?\])", webpage, 'sources')))]
+        quality = qualities(['sd', 'hd'])
+        sources = json.loads(js_to_json(self._search_regex(
+            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}\);", webpage, 'sources')))
+        formats = []
+        for container, s in sources.items():
+            for qname, video_url in s.items():
+                formats.append({
+                    'url': video_url,
+                    'container': container,
+                    'format_id': '%s-%s' % (container, qname),
+                    'quality': quality(qname),
+                })
         self._sort_formats(formats)
 
         return {

From f17e4c9c285470af3404532520599f914e73014c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 02:11:58 +0100
Subject: [PATCH 142/974] [screenwavemedia] Simplify (#3766)

---
 youtube_dl/YoutubeDL.py                 |  24 +--
 youtube_dl/extractor/__init__.py        |   2 +-
 youtube_dl/extractor/screenwavemedia.py | 220 +++++++++++-------------
 3 files changed, 113 insertions(+), 133 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 248367039..31531855e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -622,23 +622,17 @@ class YoutubeDL(object):
                 ie_result['url'], ie_key=ie_result.get('ie_key'),
                 extra_info=extra_info, download=False, process=False)
 
-            def make_result(embedded_info):
-                new_result = ie_result.copy()
-                for f in ('_type', 'url', 'ext', 'player_url', 'formats',
-                          'entries', 'ie_key', 'duration',
-                          'subtitles', 'annotations', 'format',
-                          'thumbnail', 'thumbnails'):
-                    if f in new_result:
-                        del new_result[f]
-                    if f in embedded_info:
-                        new_result[f] = embedded_info[f]
-                return new_result
-            new_result = make_result(info)
+            new_result = ie_result.copy()
+            for f in ('_type', 'id', 'url', 'ext', 'player_url', 'formats',
+                      'entries', 'ie_key', 'duration',
+                      'subtitles', 'annotations', 'format',
+                      'thumbnail', 'thumbnails'):
+                if f in new_result:
+                    del new_result[f]
+                if f in info:
+                    new_result[f] = info[f]
 
             assert new_result.get('_type') != 'url_transparent'
-            if new_result.get('_type') == 'compat_list':
-                new_result['entries'] = [
-                    make_result(e) for e in new_result['entries']]
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6b7660ab1..982a134bf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -335,7 +335,7 @@ from .savefrom import SaveFromIE
 from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
-from .screenwavemedia import ScreenwaveMediaIE
+from .screenwavemedia import CinemassacreIE, ScreenwaveMediaIE, TeamFourIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
 from .sexykarma import SexyKarmaIE
diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index cf9b8e0de..108190471 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -6,109 +6,28 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    month_by_name,
     int_or_none,
+    month_by_name,
+    unified_strdate,
 )
 
+
 class ScreenwaveMediaIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?' \
-        r':(?P<generic>player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<video_id>.+))' \
-        r'|(?P<cinemassacre>(?:www\.)?cinemassacre\.com/(?P<cm_date_Y>[0-9]{4})/(?P<cm_date_m>[0-9]{2})/(?P<cm_date_d>[0-9]{2})/(?P<cm_display_id>[^?#/]+))' \
-        r'|(?P<teamfourstar>(?:www\.)?teamfourstar\.com/video/(?P<tfs_display_id>[a-z0-9\-]+)/?)' \
-        r')'
+    _VALID_URL = r'http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
 
-    _TESTS = [
-        {
-            'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
-            'md5': 'fde81fbafaee331785f58cd6c0d46190',
-            'info_dict': {
-                'id': 'Cinemasssacre-19911',
-                'ext': 'mp4',
-                'upload_date': '20121110',
-                'title': '“Angry Video Game Nerd: The Movie” – Trailer',
-                'description': 'md5:fb87405fcb42a331742a0dce2708560b',
-            },
-        },
-        {
-            'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
-            'md5': 'd72f10cd39eac4215048f62ab477a511',
-            'info_dict': {
-                'id': 'Cinemasssacre-521be8ef82b16',
-                'ext': 'mp4',
-                'upload_date': '20131002',
-                'title': 'The Mummy’s Hand (1940)',
-            },
-        }
-    ]
-
-    def _cinemassacre_get_info(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('cm_display_id')
-
-        webpage = self._download_webpage(url, display_id)
-        video_date = mobj.group('cm_date_Y') + mobj.group('cm_date_m') + mobj.group('cm_date_d')
-        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"', webpage)
-        if not mobj:
-            raise ExtractorError('Can\'t extract embed url and video id')
-        playerdata_url = mobj.group('embed_url')
-
-        video_title = self._html_search_regex(
-            r'<title>(?P<title>.+?)\|', webpage, 'title')
-        video_description = self._html_search_regex(
-            r'<div class="entry-content">(?P<description>.+?)</div>',
-            webpage, 'description', flags=re.DOTALL, fatal=False)
-        video_thumbnail = self._og_search_thumbnail(webpage)
-
-        return {
-            'title': video_title,
-            'description': video_description,
-            'upload_date': video_date,
-            'thumbnail': video_thumbnail,
-            '_embed_url': playerdata_url,
-        }
-
-    def _teamfourstar_get_info(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('tfs_display_id')
-        webpage = self._download_webpage(url, display_id)
-
-        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"', webpage)
-        if not mobj:
-            raise ExtractorError('Can\'t extract embed url and video id')
-        playerdata_url = mobj.group('embed_url')
-
-        video_title = self._html_search_regex(
-            r'<div class="heroheadingtitle">(?P<title>.+?)</div>', webpage, 'title')
-        video_date = self._html_search_regex(
-            r'<div class="heroheadingdate">(?P<date>.+?)</div>', webpage, 'date')
-        mobj = re.match('(?P<month>[A-Z][a-z]+) (?P<day>\d+), (?P<year>\d+)', video_date)
-        video_date = '%04u%02u%02u' % (int(mobj.group('year')), month_by_name(mobj.group('month')), int(mobj.group('day')))
-        video_description = self._html_search_regex(
-            r'<div class="postcontent">(?P<description>.+?)</div>', webpage, 'description', flags=re.DOTALL)
-        video_thumbnail = self._og_search_thumbnail(webpage)
-
-        return {
-            'title': video_title,
-            'description': video_description,
-            'upload_date': video_date,
-            'thumbnail': video_thumbnail,
-            '_embed_url': playerdata_url,
-        }
-
-    def _screenwavemedia_get_info(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if not mobj:
-            raise ExtractorError('Can\'t extract embed url and video id')
-        video_id = mobj.group('video_id')
+    _TESTS = [{
+        'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',
+        'only_matching': True,
+    }]
 
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
         playerdata = self._download_webpage(url, video_id, 'Downloading player webpage')
 
         vidtitle = self._search_regex(
-            r'\'vidtitle\'\s*:\s*"([^\']+)"', playerdata, 'vidtitle').replace('\\/', '/')
+            r'\'vidtitle\'\s*:\s*"([^"]+)"', playerdata, 'vidtitle').replace('\\/', '/')
         vidurl = self._search_regex(
-            r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/')
-        pageurl = self._search_regex(
-            r'\'pageurl\'\s*:\s*"([^\']+)"', playerdata, 'pageurl', fatal=False).replace('\\/', '/')
+            r'\'vidurl\'\s*:\s*"([^"]+)"', playerdata, 'vidurl').replace('\\/', '/')
 
         videolist_url = None
 
@@ -134,61 +53,128 @@ class ScreenwaveMediaIE(InfoExtractor):
                 file_ = src.partition(':')[-1]
                 width = int_or_none(video.get('width'))
                 height = int_or_none(video.get('height'))
-                bitrate = int_or_none(video.get('system-bitrate'))
+                bitrate = int_or_none(video.get('system-bitrate'), scale=1000)
                 format = {
                     'url': baseurl + file_,
                     'format_id': src.rpartition('.')[0].rpartition('_')[-1],
                 }
                 if width or height:
                     format.update({
-                        'tbr': bitrate // 1000 if bitrate else None,
+                        'tbr': bitrate,
                         'width': width,
                         'height': height,
                     })
                 else:
                     format.update({
-                        'abr': bitrate // 1000 if bitrate else None,
+                        'abr': bitrate,
                         'vcodec': 'none',
                     })
                 formats.append(format)
-            self._sort_formats(formats)
         else:
             formats = [{
                 'url': vidurl,
             }]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': vidtitle,
             'formats': formats,
-            '_episode_page': pageurl,
         }
 
+
+class CinemassacreIE(InfoExtractor):
+    _VALID_URL = 'https?://(?:www\.)?cinemassacre\.com/(?P<date_y>[0-9]{4})/(?P<date_m>[0-9]{2})/(?P<date_d>[0-9]{2})/(?P<display_id>[^?#/]+)'
+    _TESTS = [
+        {
+            'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
+            'md5': 'fde81fbafaee331785f58cd6c0d46190',
+            'info_dict': {
+                'id': 'Cinemassacre-19911',
+                'ext': 'mp4',
+                'upload_date': '20121110',
+                'title': '“Angry Video Game Nerd: The Movie” – Trailer',
+                'description': 'md5:fb87405fcb42a331742a0dce2708560b',
+            },
+        },
+        {
+            'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
+            'md5': 'd72f10cd39eac4215048f62ab477a511',
+            'info_dict': {
+                'id': 'Cinemassacre-521be8ef82b16',
+                'ext': 'mp4',
+                'upload_date': '20131002',
+                'title': 'The Mummy’s Hand (1940)',
+            },
+        }
+    ]
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('display_id')
+        video_date = mobj.group('date_y') + mobj.group('date_m') + mobj.group('date_d')
 
-        swm_info = None
-        site_info = None
+        webpage = self._download_webpage(url, display_id)
 
-        if mobj.group('generic'):
-            swm_info = self._screenwavemedia_get_info(url)
-            url = swm_info['_episode_page']
-            mobj = re.match(self._VALID_URL, url)
+        playerdata_url = self._search_regex(
+            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+            webpage, 'player data URL')
+        video_title = self._html_search_regex(
+            r'<title>(?P<title>.+?)\|', webpage, 'title')
+        video_description = self._html_search_regex(
+            r'<div class="entry-content">(?P<description>.+?)</div>',
+            webpage, 'description', flags=re.DOTALL, fatal=False)
+        video_thumbnail = self._og_search_thumbnail(webpage)
 
-        if mobj:
-            if mobj.group('cinemassacre'):
-                site_info = self._cinemassacre_get_info(url)
-            elif mobj.group('teamfourstar'):
-                site_info = self._teamfourstar_get_info(url)
+        return {
+            '_type': 'url_transparent',
+            'display_id': display_id,
+            'title': video_title,
+            'description': video_description,
+            'upload_date': video_date,
+            'thumbnail': video_thumbnail,
+            'url': playerdata_url,
+        }
 
-        if not swm_info:
-            if site_info:
-                swm_info = self._screenwavemedia_get_info(site_info['_embed_url'])
 
-        if not swm_info:
-            raise ExtractorError("Failed to extract metadata for this URL")
+class TeamFourIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?teamfourstar\.com/video/(?P<id>[a-z0-9\-]+)/?'
+    _TEST = {
+        'url': 'http://teamfourstar.com/video/a-moment-with-tfs-episode-4/',
+        'info_dict': {
+            'id': 'TeamFourStar-5292a02f20bfa',
+            'ext': 'mp4',
+            'upload_date': '20130401',
+            'description': 'Check out this and more on our website: http://teamfourstar.com\nTFS Store: http://sharkrobot.com/team-four-star\nFollow on Twitter: http://twitter.com/teamfourstar\nLike on FB: http://facebook.com/teamfourstar',
+            'title': 'A Moment With TFS Episode 4',
+        }
+    }
 
-        if site_info:
-            swm_info.update(site_info)
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
 
-        return swm_info
+        playerdata_url = self._search_regex(
+            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+            webpage, 'player data URL')
+
+        video_title = self._html_search_regex(
+            r'<div class="heroheadingtitle">(?P<title>.+?)</div>',
+            webpage, 'title')
+        video_date = unified_strdate(self._html_search_regex(
+            r'<div class="heroheadingdate">(?P<date>.+?)</div>',
+            webpage, 'date', fatal=False))
+        video_description = self._html_search_regex(
+            r'(?s)<div class="postcontent">(?P<description>.+?)</div>',
+            webpage, 'description', fatal=False)
+        video_thumbnail = self._og_search_thumbnail(webpage)
+
+        return {
+            '_type': 'url_transparent',
+            'display_id': display_id,
+            'title': video_title,
+            'description': video_description,
+            'upload_date': video_date,
+            'thumbnail': video_thumbnail,
+            'url': playerdata_url,
+        }

From 0fa2b899d1c5affc55bfa2e65e3b0fda0de56b10 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 02:14:04 +0100
Subject: [PATCH 143/974] [Makefile] remove *.info.json in clean target

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3e1debc7e..b846331b9 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json
 
 cleanall: clean
 	rm -f youtube-dl youtube-dl.exe

From 42bdd9d0516be5b71c89c9cccde16a880a14b0b1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 02:57:36 +0100
Subject: [PATCH 144/974] [cinchcast] Add new extractor (Fixes #4428)

---
 test/test_utils.py                |  3 ++
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/cinchcast.py | 53 +++++++++++++++++++++++++++++++
 youtube_dl/extractor/generic.py   | 20 ++++++++++--
 youtube_dl/utils.py               | 17 +++++++---
 5 files changed, 88 insertions(+), 6 deletions(-)
 create mode 100644 youtube_dl/extractor/cinchcast.py

diff --git a/test/test_utils.py b/test/test_utils.py
index aaa293ff8..d42df6d96 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -144,6 +144,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011')
         self.assertEqual(unified_strdate('1968-12-10'), '19681210')
         self.assertEqual(unified_strdate('28/01/2014 21:00:00 +0100'), '20140128')
+        self.assertEqual(
+            unified_strdate('11/26/2014 11:30:00 AM PST', day_first=False),
+            '20141126')
 
     def test_find_xpath_attr(self):
         testxml = '''<root>
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 982a134bf..746ee69e4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -51,6 +51,7 @@ from .cbsnews import CBSNewsIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
+from .cinchcast import CinchcastIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .clipsyndicate import ClipsyndicateIE
diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dl/extractor/cinchcast.py
new file mode 100644
index 000000000..8ce8b3128
--- /dev/null
+++ b/youtube_dl/extractor/cinchcast.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+    xpath_text,
+)
+
+
+class CinchcastIE(InfoExtractor):
+    _VALID_URL = r'https?://player\.cinchcast\.com/.*?assetId=(?P<id>[0-9]+)'
+    _TEST = {
+        # Actual test is run in generic, look for undergroundwellness
+        'url': 'http://player.cinchcast.com/?platformId=1&#038;assetType=single&#038;assetId=7141703',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        doc = self._download_xml(
+            'http://www.blogtalkradio.com/playerasset/mrss?assetType=single&assetId=%s' % video_id,
+            video_id)
+
+        item = doc.find('.//item')
+        title = xpath_text(item, './title', fatal=True)
+        date_str = xpath_text(
+            item, './{http://developer.longtailvideo.com/trac/}date')
+        upload_date = unified_strdate(date_str, day_first=False)
+        # duration is present but wrong
+        formats = []
+        formats.append({
+            'format_id': 'main',
+            'url': item.find(
+                './{http://search.yahoo.com/mrss/}content').attrib['url'],
+        })
+        backup_url = xpath_text(
+            item, './{http://developer.longtailvideo.com/trac/}backupContent')
+        if backup_url:
+            formats.append({
+                'preference': 2,  # seems to be more reliable
+                'format_id': 'backup',
+                'url': backup_url,
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'upload_date': upload_date,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 328301de3..2b4d8c62f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -467,8 +467,17 @@ class GenericIE(InfoExtractor):
             'expected_warnings': [
                 'URL could be a direct video link, returning it as such.'
             ]
-        }
-
+        },
+        # Cinchcast embed
+        {
+            'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
+            'info_dict': {
+                'id': '7141703',
+                'ext': 'mp3',
+                'upload_date': '20141126',
+                'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
+            }
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -962,6 +971,13 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'SBS')
 
+        # Look for embedded Cinchcast player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
+            webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Cinchcast')
+
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
             webpage)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 75f9594e6..4b0567c93 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -166,7 +166,7 @@ def xpath_text(node, xpath, name=None, fatal=False):
         xpath = xpath.encode('ascii')
 
     n = node.find(xpath)
-    if n is None:
+    if n is None or n.text is None:
         if fatal:
             name = xpath if name is None else name
             raise ExtractorError('Could not find XML element %s' % name)
@@ -644,17 +644,19 @@ def parse_iso8601(date_str, delimiter='T'):
     return calendar.timegm(dt.timetuple())
 
 
-def unified_strdate(date_str):
+def unified_strdate(date_str, day_first=True):
     """Return a string with the date in the format YYYYMMDD"""
 
     if date_str is None:
         return None
-
     upload_date = None
     # Replace commas
     date_str = date_str.replace(',', ' ')
     # %z (UTC offset) is only supported in python>=3.2
     date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
+    # Remove AM/PM + timezone
+    date_str = re.sub(r'(?i)\s*(?:AM|PM)\s+[A-Z]+', '', date_str)
+
     format_expressions = [
         '%d %B %Y',
         '%d %b %Y',
@@ -669,7 +671,6 @@ def unified_strdate(date_str):
         '%d/%m/%Y',
         '%d/%m/%y',
         '%Y/%m/%d %H:%M:%S',
-        '%d/%m/%Y %H:%M:%S',
         '%Y-%m-%d %H:%M:%S',
         '%Y-%m-%d %H:%M:%S.%f',
         '%d.%m.%Y %H:%M',
@@ -681,6 +682,14 @@ def unified_strdate(date_str):
         '%Y-%m-%dT%H:%M:%S.%f',
         '%Y-%m-%dT%H:%M',
     ]
+    if day_first:
+        format_expressions.extend([
+            '%d/%m/%Y %H:%M:%S',
+        ])
+    else:
+        format_expressions.extend([
+            '%m/%d/%Y %H:%M:%S',
+        ])
     for expression in format_expressions:
         try:
             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')

From 63e0f295645c768420bdf19b598393799c108652 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 02:59:52 +0100
Subject: [PATCH 145/974] [vine] Modernize

---
 youtube_dl/extractor/vine.py | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 4970b2f23..ef3dadeb5 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -25,26 +25,21 @@ class VineIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage('https://vine.co/v/' + video_id, video_id)
 
         data = json.loads(self._html_search_regex(
             r'window\.POST_DATA = { %s: ({.+?}) }' % video_id, webpage, 'vine data'))
 
-        formats = [
-            {
-                'url': data['videoLowURL'],
-                'ext': 'mp4',
-                'format_id': 'low',
-            },
-            {
-                'url': data['videoUrl'],
-                'ext': 'mp4',
-                'format_id': 'standard',
-            }
-        ]
+        formats = [{
+            'url': data['videoLowURL'],
+            'ext': 'mp4',
+            'format_id': 'low',
+        }, {
+            'url': data['videoUrl'],
+            'ext': 'mp4',
+            'format_id': 'standard',
+        }]
 
         return {
             'id': video_id,

From 6a5308ab499ef656fca65b6f22fd02c47486d94e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 03:02:56 +0100
Subject: [PATCH 146/974] release 2014.12.12

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b2a285bb2..eda34d8d1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.11'
+__version__ = '2014.12.12'

From f5e43bc695947989d5d563f53135fb8fcfcd4081 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 03:34:28 +0100
Subject: [PATCH 147/974] [vine] Provide alt_title (Fixes #4448)

---
 youtube_dl/extractor/common.py | 3 ++-
 youtube_dl/extractor/vine.py   | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2277ec6ab..c2e24def1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -118,6 +118,7 @@ class InfoExtractor(object):
 
     The following fields are optional:
 
+    alt_title:      A secondary title of the video.
     display_id      An alternative identifier for the video, not necessarily
                     unique, but available before title. Typically, id is
                     something like "4234987", title "Dancing naked mole rats",
@@ -129,7 +130,7 @@ class InfoExtractor(object):
                         * "resolution" (optional, string "{width}x{height"},
                                         deprecated)
     thumbnail:      Full URL to a video thumbnail image.
-    description:    One-line video description.
+    description:    Full video description.
     uploader:       Full name of the video uploader.
     timestamp:      UNIX timestamp of the moment the video became available.
     upload_date:    Video upload date (YYYYMMDD).
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index ef3dadeb5..0b58fe0fe 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -17,6 +17,7 @@ class VineIE(InfoExtractor):
             'id': 'b9KOOWX7HUx',
             'ext': 'mp4',
             'title': 'Chicken.',
+            'alt_title': 'Vine by Jack Dorsey',
             'description': 'Chicken.',
             'upload_date': '20130519',
             'uploader': 'Jack Dorsey',
@@ -44,6 +45,7 @@ class VineIE(InfoExtractor):
         return {
             'id': video_id,
             'title': self._og_search_title(webpage),
+            'alt_title': self._og_search_description(webpage),
             'description': data['description'],
             'thumbnail': data['thumbnailUrl'],
             'upload_date': unified_strdate(data['created']),

From d068ba24f3fa247b262a0aed6d94ac7f4f43de97 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 03:34:33 +0100
Subject: [PATCH 148/974] release 2014.12.12.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index eda34d8d1..7289ea4d2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12'
+__version__ = '2014.12.12.1'

From 16040f46d64bad8dcc5f948288ef469dd787d3d3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 04:01:08 +0100
Subject: [PATCH 149/974] [utils] Work around PyPy stupidity with Windows DLLs
 (Fixes #4392)

---
 youtube_dl/compat.py | 19 +++++++++++++++++++
 youtube_dl/utils.py  |  9 +++++----
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f4a85443e..cd46693b3 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
+import ctypes
 import getpass
 import optparse
 import os
+import platform
 import re
 import subprocess
 import sys
@@ -326,6 +328,22 @@ def workaround_optparse_bug9161():
         optparse.OptionGroup.add_option = _compat_add_option
 
 
+if platform.python_implementation() == 'PyPy':
+    # PyPy expects byte strings as Windows function names
+    # https://github.com/rg3/youtube-dl/pull/4392
+    def compat_WINFUNCTYPE(*args, **kwargs):
+        real = ctypes.WINFUNCTYPE(*args, **kwargs)
+
+        def resf(tpl, *args, **kwargs):
+            funcname, dll = tpl
+            return real((str(funcname), dll), *args, **kwargs)
+
+        return resf
+else:
+    def compat_WINFUNCTYPE(*args, **kwargs):
+        return ctypes.WINFUNCTYPE(*args, **kwargs)
+
+
 __all__ = [
     'compat_HTTPError',
     'compat_chr',
@@ -349,6 +367,7 @@ __all__ = [
     'compat_urllib_request',
     'compat_urlparse',
     'compat_urlretrieve',
+    'compat_WINFUNCTYPE',
     'compat_xml_parse_error',
     'shlex_quote',
     'subprocess_check_output',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4b0567c93..bbe554a65 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -41,6 +41,7 @@ from .compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
+    compat_WINFUNCTYPE,
     shlex_quote,
 )
 
@@ -817,21 +818,21 @@ def _windows_write_string(s, out):
     if fileno not in WIN_OUTPUT_IDS:
         return False
 
-    GetStdHandle = ctypes.WINFUNCTYPE(
+    GetStdHandle = compat_WINFUNCTYPE(
         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
         ("GetStdHandle", ctypes.windll.kernel32))
     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
 
-    WriteConsoleW = ctypes.WINFUNCTYPE(
+    WriteConsoleW = compat_WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
         ctypes.wintypes.LPVOID)(("WriteConsoleW", ctypes.windll.kernel32))
     written = ctypes.wintypes.DWORD(0)
 
-    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(("GetFileType", ctypes.windll.kernel32))
+    GetFileType = compat_WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(("GetFileType", ctypes.windll.kernel32))
     FILE_TYPE_CHAR = 0x0002
     FILE_TYPE_REMOTE = 0x8000
-    GetConsoleMode = ctypes.WINFUNCTYPE(
+    GetConsoleMode = compat_WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
         ctypes.POINTER(ctypes.wintypes.DWORD))(
         ("GetConsoleMode", ctypes.windll.kernel32))

From 4f978523161abb0af241a91da13158d6d91ff442 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 04:09:32 +0100
Subject: [PATCH 150/974] Remove unused imports

---
 youtube_dl/extractor/cinchcast.py       | 1 -
 youtube_dl/extractor/pornhd.py          | 1 -
 youtube_dl/extractor/screenwavemedia.py | 2 --
 youtube_dl/extractor/tvplay.py          | 1 -
 4 files changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dl/extractor/cinchcast.py
index 8ce8b3128..0c9a24bef 100644
--- a/youtube_dl/extractor/cinchcast.py
+++ b/youtube_dl/extractor/cinchcast.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
     unified_strdate,
     xpath_text,
 )
diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 63472b33e..954dfccb7 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -8,7 +8,6 @@ from ..utils import (
     int_or_none,
     js_to_json,
     qualities,
-    determine_ext,
 )
 
 
diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 108190471..6c9fdb7c1 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -5,9 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
     int_or_none,
-    month_by_name,
     unified_strdate,
 )
 
diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 0157392cc..9a53a3c74 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -6,7 +6,6 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
     parse_iso8601,
     qualities,
 )

From 62651c556a1c61b5e73a958e2bc5c1c0624213bc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 04:23:34 +0100
Subject: [PATCH 151/974] [howstuffworks] Parse only once, but right (#4383)

---
 youtube_dl/extractor/howstuffworks.py | 53 ++++++++++++++-------------
 1 file changed, 28 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index 9478e9e67..e97339121 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -1,7 +1,12 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import find_xpath_attr
+from ..utils import (
+    find_xpath_attr,
+    int_or_none,
+    js_to_json,
+    unescapeHTML,
+)
 
 
 class HowStuffWorksIE(InfoExtractor):
@@ -14,7 +19,9 @@ class HowStuffWorksIE(InfoExtractor):
                 'ext': 'flv',
                 'title': 'Cool Jobs - Iditarod Musher',
                 'description': 'Cold sleds, freezing temps and warm dog breath... an Iditarod musher\'s dream. Kasey-Dee Gardner jumps on a sled to find out what the big deal is.',
-                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/5266.jpg',
+                'display_id': 'cool-jobs-iditarod-musher',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 161,
             },
         },
         {
@@ -24,7 +31,8 @@ class HowStuffWorksIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Survival Zone: Food and Water In the Savanna',
                 'description': 'Learn how to find both food and water while trekking in the African savannah. In this video from the Discovery Channel.',
-                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/7199.jpg',
+                'display_id': 'survival-zone-food-and-water-in-the-savanna',
+                'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
         {
@@ -34,7 +42,8 @@ class HowStuffWorksIE(InfoExtractor):
                 'ext': 'flv',
                 'title': 'Sword Swallowing #1 by Dan Meyer',
                 'description': 'Video footage (1 of 3) used by permission of the owner Dan Meyer through Sword Swallowers Association International <www.swordswallow.org>',
-                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/118306353233.jpg',
+                'display_id': 'sword-swallowing-1-by-dan-meyer',
+                'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
     ]
@@ -42,23 +51,17 @@ class HowStuffWorksIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        clip_info = self._search_regex('(?s)var clip = {(.*?)};', webpage, 'clip info')
+        clip_js = self._search_regex(
+            r'(?s)var clip = ({.*?});', webpage, 'clip info')
+        clip_info = self._parse_json(
+            clip_js, display_id, transform_source=js_to_json)
 
-        def extract_clip_info(key, clip_info, name=None, **kargs):
-            if name is None:
-                name = key
-            return self._html_search_regex(
-                r"\s*%s\s*: '?([^'\n]*[^'\n,])" % key, clip_info, name, **kargs)
-
-        video_id = extract_clip_info('content_id', clip_info, 'video id')
+        video_id = clip_info['content_id']
         formats = []
-        m3u8_url = extract_clip_info('m3u8', clip_info, 'm3u8 url', default=None)
-        if m3u8_url is not None:
-            formats += self._extract_m3u8_formats(m3u8_url , video_id, 'mp4')
-        mp4 = self._parse_json(
-            extract_clip_info(
-                'mp4', clip_info, 'formats').replace('},]','}]'), video_id)
-        for video in mp4:
+        m3u8_url = clip_info.get('m3u8')
+        if m3u8_url:
+            formats += self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+        for video in clip_info.get('mp4', []):
             formats.append({
                 'url': video['src'],
                 'format_id': video['bitrate'],
@@ -80,7 +83,7 @@ class HowStuffWorksIE(InfoExtractor):
 
             for video in smil.findall(
                     './{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')):
-                vbr = int(video.attrib['system-bitrate']) / 1000
+                vbr = int_or_none(video.attrib['system-bitrate'], scale=1000)
                 formats.append({
                     'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX),
                     'format_id': '%dk' % vbr,
@@ -90,11 +93,11 @@ class HowStuffWorksIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
+            'id': '%s' % video_id,
             'display_id': display_id,
-            'title': extract_clip_info('clip_title', clip_info, 'title'),
-            'description': extract_clip_info('caption', clip_info, 'description', fatal=False),
-            'thumbnail': extract_clip_info('video_still_url', clip_info, 'thumbnail'),
-            'duration': extract_clip_info('duration', clip_info),
+            'title': unescapeHTML(clip_info['clip_title']),
+            'description': unescapeHTML(clip_info.get('caption')),
+            'thumbnail': clip_info.get('video_still_url'),
+            'duration': clip_info.get('duration'),
             'formats': formats,
         }

From e2f89ec7aada2a58ca658a77401762b764adc112 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 11:33:55 +0100
Subject: [PATCH 152/974] Revert "[utils] Work around PyPy stupidity with
 Windows DLLs (Fixes #4392)"

This reverts commit 16040f46d64bad8dcc5f948288ef469dd787d3d3.
---
 youtube_dl/compat.py | 19 -------------------
 youtube_dl/utils.py  |  9 ++++-----
 2 files changed, 4 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index cd46693b3..f4a85443e 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 
-import ctypes
 import getpass
 import optparse
 import os
-import platform
 import re
 import subprocess
 import sys
@@ -328,22 +326,6 @@ def workaround_optparse_bug9161():
         optparse.OptionGroup.add_option = _compat_add_option
 
 
-if platform.python_implementation() == 'PyPy':
-    # PyPy expects byte strings as Windows function names
-    # https://github.com/rg3/youtube-dl/pull/4392
-    def compat_WINFUNCTYPE(*args, **kwargs):
-        real = ctypes.WINFUNCTYPE(*args, **kwargs)
-
-        def resf(tpl, *args, **kwargs):
-            funcname, dll = tpl
-            return real((str(funcname), dll), *args, **kwargs)
-
-        return resf
-else:
-    def compat_WINFUNCTYPE(*args, **kwargs):
-        return ctypes.WINFUNCTYPE(*args, **kwargs)
-
-
 __all__ = [
     'compat_HTTPError',
     'compat_chr',
@@ -367,7 +349,6 @@ __all__ = [
     'compat_urllib_request',
     'compat_urlparse',
     'compat_urlretrieve',
-    'compat_WINFUNCTYPE',
     'compat_xml_parse_error',
     'shlex_quote',
     'subprocess_check_output',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bbe554a65..4b0567c93 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -41,7 +41,6 @@ from .compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
-    compat_WINFUNCTYPE,
     shlex_quote,
 )
 
@@ -818,21 +817,21 @@ def _windows_write_string(s, out):
     if fileno not in WIN_OUTPUT_IDS:
         return False
 
-    GetStdHandle = compat_WINFUNCTYPE(
+    GetStdHandle = ctypes.WINFUNCTYPE(
         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
         ("GetStdHandle", ctypes.windll.kernel32))
     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
 
-    WriteConsoleW = compat_WINFUNCTYPE(
+    WriteConsoleW = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
         ctypes.wintypes.LPVOID)(("WriteConsoleW", ctypes.windll.kernel32))
     written = ctypes.wintypes.DWORD(0)
 
-    GetFileType = compat_WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(("GetFileType", ctypes.windll.kernel32))
+    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(("GetFileType", ctypes.windll.kernel32))
     FILE_TYPE_CHAR = 0x0002
     FILE_TYPE_REMOTE = 0x8000
-    GetConsoleMode = compat_WINFUNCTYPE(
+    GetConsoleMode = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
         ctypes.POINTER(ctypes.wintypes.DWORD))(
         ("GetConsoleMode", ctypes.windll.kernel32))

From c6973bd4124bb7e15b937f2d337bd7af20562ca6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 12:42:33 +0100
Subject: [PATCH 153/974] [compat] Simplify kwarg detection code

This enables nuitka to compile youtube-dl.
---
 youtube_dl/compat.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f4a85443e..46d438846 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -297,7 +297,9 @@ else:
 
 # Old 2.6 and 2.7 releases require kwargs to be bytes
 try:
-    (lambda x: x)(**{'x': 0})
+    def _testfunc(x):
+        pass
+    _testfunc(**{'x': 0})
 except TypeError:
     def compat_kwargs(kwargs):
         return dict((bytes(k), v) for k, v in kwargs.items())

From 73aeb2dc56068f50e996f2cc08c58db9258b21d4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 12:44:32 +0100
Subject: [PATCH 154/974] [goshgay] Modernize

---
 youtube_dl/extractor/goshgay.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/goshgay.py b/youtube_dl/extractor/goshgay.py
index 18474cbb7..ccf65c2c3 100644
--- a/youtube_dl/extractor/goshgay.py
+++ b/youtube_dl/extractor/goshgay.py
@@ -2,14 +2,14 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
-    compat_urlparse,
     ExtractorError,
 )
 
 
 class GoshgayIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)www.goshgay.com/video(?P<id>\d+?)($|/)'
+    _VALID_URL = r'https?://www\.goshgay\.com/video(?P<id>\d+?)($|/)'
     _TEST = {
         'url': 'http://www.goshgay.com/video4116282',
         'md5': '268b9f3c3229105c57859e166dd72b03',
@@ -24,8 +24,8 @@ class GoshgayIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
+
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
         family_friendly = self._html_search_meta(

From 82b34105d301f0aa6897b6695e390324e3722add Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 12:55:13 +0100
Subject: [PATCH 155/974] [goshgay] Fix extraction

---
 youtube_dl/extractor/goshgay.py | 47 +++++++++++++++------------------
 1 file changed, 21 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/goshgay.py b/youtube_dl/extractor/goshgay.py
index ccf65c2c3..b116d251d 100644
--- a/youtube_dl/extractor/goshgay.py
+++ b/youtube_dl/extractor/goshgay.py
@@ -2,22 +2,25 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_parse_qs,
+)
 from ..utils import (
-    ExtractorError,
+    parse_duration,
 )
 
 
 class GoshgayIE(InfoExtractor):
     _VALID_URL = r'https?://www\.goshgay\.com/video(?P<id>\d+?)($|/)'
     _TEST = {
-        'url': 'http://www.goshgay.com/video4116282',
-        'md5': '268b9f3c3229105c57859e166dd72b03',
+        'url': 'http://www.goshgay.com/video299069/diesel_sfw_xxx_video',
+        'md5': '027fcc54459dff0feb0bc06a7aeda680',
         'info_dict': {
-            'id': '4116282',
+            'id': '299069',
             'ext': 'flv',
-            'title': 'md5:089833a4790b5e103285a07337f245bf',
-            'thumbnail': 're:http://.*\.jpg',
+            'title': 'DIESEL SFW XXX Video',
+            'thumbnail': 're:^http://.*\.jpg$',
+            'duration': 79,
             'age_limit': 18,
         }
     }
@@ -26,33 +29,25 @@ class GoshgayIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._og_search_title(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
+        title = self._html_search_regex(
+            r'<h2>(.*?)<', webpage, 'title')
+        duration = parse_duration(self._html_search_regex(
+            r'<span class="duration">\s*-?\s*(.*?)</span>',
+            webpage, 'duration', fatal=False))
         family_friendly = self._html_search_meta(
             'isFamilyFriendly', webpage, default='false')
-        config_url = self._search_regex(
-            r"'config'\s*:\s*'([^']+)'", webpage, 'config URL')
 
-        config = self._download_xml(
-            config_url, video_id, 'Downloading player config XML')
-
-        if config is None:
-            raise ExtractorError('Missing config XML')
-        if config.tag != 'config':
-            raise ExtractorError('Missing config attribute')
-        fns = config.findall('file')
-        if len(fns) < 1:
-            raise ExtractorError('Missing media URI')
-        video_url = fns[0].text
-
-        url_comp = compat_urlparse.urlparse(url)
-        ref = "%s://%s%s" % (url_comp[0], url_comp[1], url_comp[2])
+        flashvars = compat_parse_qs(self._html_search_regex(
+            r'<embed.+?id="flash-player-embed".+?flashvars="([^"]+)"',
+            webpage, 'flashvars'))
+        thumbnail = flashvars.get('url_bigthumb', [None])[0]
+        video_url = flashvars['flv_url'][0]
 
         return {
             'id': video_id,
             'url': video_url,
             'title': title,
             'thumbnail': thumbnail,
-            'http_referer': ref,
+            'duration': duration,
             'age_limit': 0 if family_friendly == 'true' else 18,
         }

From 0d2fb1d19342aa1f9ae2f9509ca3cdb819c6b8d1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 13:03:16 +0100
Subject: [PATCH 156/974] [helsinki] Fix extraction

---
 youtube_dl/extractor/helsinki.py | 37 ++++++++------------------------
 1 file changed, 9 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/helsinki.py b/youtube_dl/extractor/helsinki.py
index 5268efa49..93107b306 100644
--- a/youtube_dl/extractor/helsinki.py
+++ b/youtube_dl/extractor/helsinki.py
@@ -2,9 +2,8 @@
 
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import js_to_json
 
 
 class HelsinkiIE(InfoExtractor):
@@ -24,39 +23,21 @@ class HelsinkiIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        formats = []
-
-        mobj = re.search(r'file=((\w+):[^&]+)', webpage)
-        if mobj:
-            formats.append({
-                'ext': mobj.group(2),
-                'play_path': mobj.group(1),
-                'url': 'rtmp://flashvideo.it.helsinki.fi/vod/',
-                'player_url': 'http://video.helsinki.fi/player.swf',
-                'format_note': 'sd',
-                'quality': 0,
-            })
-
-        mobj = re.search(r'hd\.file=((\w+):[^&]+)', webpage)
-        if mobj:
-            formats.append({
-                'ext': mobj.group(2),
-                'play_path': mobj.group(1),
-                'url': 'rtmp://flashvideo.it.helsinki.fi/vod/',
-                'player_url': 'http://video.helsinki.fi/player.swf',
-                'format_note': 'hd',
-                'quality': 1,
-            })
 
+        params = self._parse_json(self._html_search_regex(
+            r'(?s)jwplayer\("player"\).setup\((\{.*?\})\);',
+            webpage, 'player code'), video_id, transform_source=js_to_json)
+        formats = [{
+            'url': s['file'],
+            'ext': 'mp4',
+        } for s in params['sources']]
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': self._og_search_title(webpage).replace('Video: ', ''),
             'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
         }

From 025f30ba38311cfffef7bafa31ae87b8365b47bd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 13:07:43 +0100
Subject: [PATCH 157/974] [channel9] Do not return compat_list results anymore

---
 youtube_dl/extractor/channel9.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 2a05813f8..3dfc24f5b 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -236,16 +236,17 @@ class Channel9IE(InfoExtractor):
         if contents is None:
             return contents
 
-        session_meta = {'session_code': self._extract_session_code(html),
-                        'session_day': self._extract_session_day(html),
-                        'session_room': self._extract_session_room(html),
-                        'session_speakers': self._extract_session_speakers(html),
-                        }
+        session_meta = {
+            'session_code': self._extract_session_code(html),
+            'session_day': self._extract_session_day(html),
+            'session_room': self._extract_session_room(html),
+            'session_speakers': self._extract_session_speakers(html),
+        }
 
         for content in contents:
             content.update(session_meta)
 
-        return contents
+        return self.playlist_result(contents)
 
     def _extract_list(self, content_path):
         rss = self._download_xml(self._RSS_URL % content_path, content_path, 'Downloading RSS')

From 751536f5c80cdfdbf6ddfd9f9a673ffd3d8f5d55 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 13:09:54 +0100
Subject: [PATCH 158/974] [goldenmoustache] Remove view count

view count is not present anymore, so we can't extract it.
---
 youtube_dl/extractor/goldenmoustache.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/goldenmoustache.py b/youtube_dl/extractor/goldenmoustache.py
index 10001d4d9..4a07b5558 100644
--- a/youtube_dl/extractor/goldenmoustache.py
+++ b/youtube_dl/extractor/goldenmoustache.py
@@ -17,7 +17,6 @@ class GoldenMoustacheIE(InfoExtractor):
             'title': 'Suricate - Le Poker',
             'description': 'md5:3d1f242f44f8c8cb0a106f1fd08e5dc9',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'view_count': int,
         }
     }, {
         'url': 'http://www.goldenmoustache.com/le-lab-tout-effacer-mc-fly-et-carlito-55249/',
@@ -28,7 +27,6 @@ class GoldenMoustacheIE(InfoExtractor):
             'title': 'Le LAB - Tout Effacer (Mc Fly et Carlito)',
             'description': 'md5:9b7fbf11023fb2250bd4b185e3de3b2a',
             'thumbnail': 're:^https?://.*\.(?:png|jpg)$',
-            'view_count': int,
         }
     }]
 
@@ -42,9 +40,6 @@ class GoldenMoustacheIE(InfoExtractor):
             r'<title>(.*?)(?: - Golden Moustache)?</title>', webpage, 'title')
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage)
-        view_count = int_or_none(self._html_search_regex(
-            r'<strong>([0-9]+)</strong>\s*VUES</span>',
-            webpage, 'view count', fatal=False))
 
         return {
             'id': video_id,
@@ -53,5 +48,4 @@ class GoldenMoustacheIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'view_count': view_count,
         }

From 412c617d0f7d22e9b2f7c3b0252a76bcf8ffef2f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 15:55:55 +0100
Subject: [PATCH 159/974] [cnet] Update to new theplatform infrastructure
 (Fixes #2736)

---
 youtube_dl/YoutubeDL.py      | 16 +++++++---------
 youtube_dl/extractor/cnet.py | 33 ++++++++++++++-------------------
 2 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 31531855e..e7abb7e0b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -622,15 +622,13 @@ class YoutubeDL(object):
                 ie_result['url'], ie_key=ie_result.get('ie_key'),
                 extra_info=extra_info, download=False, process=False)
 
-            new_result = ie_result.copy()
-            for f in ('_type', 'id', 'url', 'ext', 'player_url', 'formats',
-                      'entries', 'ie_key', 'duration',
-                      'subtitles', 'annotations', 'format',
-                      'thumbnail', 'thumbnails'):
-                if f in new_result:
-                    del new_result[f]
-                if f in info:
-                    new_result[f] = info[f]
+            force_properties = dict(
+                (k, v) for k, v in ie_result.items() if v is not None)
+            for f in ('_type', 'url'):
+                if f in force_properties:
+                    del force_properties[f]
+            new_result = info.copy()
+            new_result.update(force_properties)
 
             assert new_result.get('_type') != 'url_transparent'
 
diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 710d5009b..a6f1f3a91 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -15,23 +15,24 @@ class CNETIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?cnet\.com/videos/(?P<id>[^/]+)/'
     _TEST = {
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
-        'md5': '041233212a0d06b179c87cbcca1577b8',
         'info_dict': {
             'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Hands-on with Microsoft Windows 8.1 Update',
             'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
             'thumbnail': 're:^http://.*/flmswindows8.jpg$',
-            'uploader_id': 'sarah.mitroff@cbsinteractive.com',
+            'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
             'uploader': 'Sarah Mitroff',
+        },
+        'params': {
+            'skip_download': 'requires rtmpdump',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-
+        display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
+
         data_json = self._html_search_regex(
             r"<div class=\"cnetVideoPlayer\"\s+.*?data-cnet-video-options='([^']+)'",
             webpage, 'data json')
@@ -42,37 +43,31 @@ class CNETIE(InfoExtractor):
         if not vdata:
             raise ExtractorError('Cannot find video data')
 
+        mpx_account = data['config']['players']['default']['mpx_account']
+        vid = vdata['files']['rtmp']
+        tp_link = 'http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)
+
         video_id = vdata['id']
         title = vdata.get('headline')
         if title is None:
             title = vdata.get('title')
         if title is None:
             raise ExtractorError('Cannot find title!')
-        description = vdata.get('dek')
         thumbnail = vdata.get('image', {}).get('path')
         author = vdata.get('author')
         if author:
             uploader = '%s %s' % (author['firstName'], author['lastName'])
-            uploader_id = author.get('email')
+            uploader_id = author.get('id')
         else:
             uploader = None
             uploader_id = None
 
-        formats = [{
-            'format_id': '%s-%s-%s' % (
-                f['type'], f['format'],
-                int_or_none(f.get('bitrate'), 1000, default='')),
-            'url': f['uri'],
-            'tbr': int_or_none(f.get('bitrate'), 1000),
-        } for f in vdata['files']['data']]
-        self._sort_formats(formats)
-
         return {
+            '_type': 'url_transparent',
+            'url': tp_link,
             'id': video_id,
             'display_id': display_id,
             'title': title,
-            'formats': formats,
-            'description': description,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'thumbnail': thumbnail,

From 3b0bec8d11cdd4ef17a4d53295ad2717193d5308 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 15:56:45 +0100
Subject: [PATCH 160/974] release 2014.12.12.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7289ea4d2..61c106e1a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.1'
+__version__ = '2014.12.12.2'

From 68d301ffd4dba86b06fb7cc54634b5762df14318 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Dec 2014 21:23:42 +0600
Subject: [PATCH 161/974] [giantbomb] Add extractor (Closes #4432)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/giantbomb.py | 81 +++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)
 create mode 100644 youtube_dl/extractor/giantbomb.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 746ee69e4..6866f06d4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -148,6 +148,7 @@ from .gamestar import GameStarIE
 from .gametrailers import GametrailersIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
+from .giantbomb import GiantBombIE
 from .glide import GlideIE
 from .globo import GloboIE
 from .godtube import GodTubeIE
diff --git a/youtube_dl/extractor/giantbomb.py b/youtube_dl/extractor/giantbomb.py
new file mode 100644
index 000000000..87cd19147
--- /dev/null
+++ b/youtube_dl/extractor/giantbomb.py
@@ -0,0 +1,81 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    unescapeHTML,
+    qualities,
+    int_or_none,
+)
+
+
+class GiantBombIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?giantbomb\.com/videos/(?P<display_id>[^/]+)/(?P<id>\d+-\d+)'
+    _TEST = {
+        'url': 'http://www.giantbomb.com/videos/quick-look-destiny-the-dark-below/2300-9782/',
+        'md5': '57badeface303ecf6b98b812de1b9018',
+        'info_dict': {
+            'id': '2300-9782',
+            'display_id': 'quick-look-destiny-the-dark-below',
+            'ext': 'mp4',
+            'title': 'Quick Look: Destiny: The Dark Below',
+            'description': 'md5:0aa3aaf2772a41b91d44c63f30dfad24',
+            'duration': 2399,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        video = json.loads(unescapeHTML(self._search_regex(
+            r'data-video="([^"]+)"', webpage, 'data-video')))
+
+        duration = int_or_none(video.get('lengthSeconds'))
+
+        quality = qualities([
+            'f4m_low', 'progressive_low', 'f4m_high',
+            'progressive_high', 'f4m_hd', 'progressive_hd'])
+
+        formats = []
+        for format_id, video_url in video['videoStreams'].items():
+            if format_id == 'f4m_stream':
+                continue
+            if video_url.endswith('.f4m'):
+                f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.3.1', display_id)
+                if f4m_formats:
+                    f4m_formats[0]['quality'] = quality(format_id)
+                    formats.extend(f4m_formats)
+            else:
+                formats.append({
+                    'url': video_url,
+                    'format_id': format_id,
+                    'quality': quality(format_id),
+                })
+
+        if not formats:
+            youtube_id = video.get('youtubeID')
+            if youtube_id:
+                return self.url_result(youtube_id, 'Youtube')
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }

From b3efb3ebae6bcedfb972e45c7266a31afa5591c6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 16:40:37 +0100
Subject: [PATCH 162/974] [README] More concise and nicer bug reporting
 instructions

---
 README.md | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 18be078d7..b3af18912 100644
--- a/README.md
+++ b/README.md
@@ -536,13 +536,11 @@ Most likely, you'll want to use various options. For a list of what can be done,
 
 # BUGS
 
-Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email.
+Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode.
 
 Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
-For discussions, join us in the irc channel #youtube-dl on freenode.
-
-When you submit a request, please re-read it once to avoid a couple of mistakes (you can and should use this as a checklist):
+Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 
 ### Is the description of the issue itself sufficient?
 

From e56190b378367291abe4677f2357fcfc213b1a6a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 16:42:40 +0100
Subject: [PATCH 163/974] [Makefile] Add CONTRIBUTING.md (Fixes #2984)

---
 CONTRIBUTING.md                 | 150 ++++++++++++++++++++++++++++++++
 Makefile                        |   7 +-
 devscripts/make_contributing.py |  31 +++++++
 3 files changed, 186 insertions(+), 2 deletions(-)
 create mode 100644 CONTRIBUTING.md
 create mode 100755 devscripts/make_contributing.py

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 000000000..48840f8f6
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,150 @@
+Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+
+Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
+
+### Is the description of the issue itself sufficient?
+
+We often get issue reports that we cannot really decipher. While in most cases we eventually get the required information after asking back multiple times, this poses an unnecessary drain on our resources. Many contributors, including myself, are also not native speakers, so we may misread some parts.
+
+So please elaborate on what feature you are requesting, or what bug you want to be fixed. Make sure that it's obvious
+
+- What the problem is
+- How it could be fixed
+- How your proposed solution would look like
+
+If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
+
+For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+
+Site support requests **must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
+
+###  Are you using the latest version?
+
+Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
+
+###  Is the issue already documented?
+
+Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or at https://github.com/rg3/youtube-dl/search?type=Issues . If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
+
+###  Why are existing options not enough?
+
+Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/rg3/youtube-dl/blob/master/README.md#synopsis). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem.
+
+###  Is there enough context in your bug report?
+
+People want to solve problems, and often think they do us a favor by breaking down their larger problems (e.g. wanting to skip already downloaded files) to a specific request (e.g. requesting us to look whether the file exists before downloading the info page). However, what often happens is that they break down the problem into two steps: One simple, and one impossible (or extremely complicated one).
+
+We are then presented with a very complicated request when the original problem could be solved far easier, e.g. by recording the downloaded video IDs in a separate file. To avoid this, you must include the greater context where it is non-obvious. In particular, every feature request that does not consist of adding support for a new site should contain a use case scenario that explains in what situation the missing feature would be useful.
+
+###  Does the issue involve one problem, and one problem only?
+
+Some of our users seem to think there is a limit of issues they can or should open. There is no limit of issues they can or should open. While it may seem appealing to be able to dump all your issues into one ticket, that means that someone who solves one of your issues cannot mark the issue as closed. Typically, reporting a bunch of issues leads to the ticket lingering since nobody wants to attack that behemoth, until someone mercifully splits the issue into multiple ones.
+
+In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of youtube-dl that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service.
+
+###  Is anyone going to need the feature?
+
+Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
+
+###  Is your question about youtube-dl?
+
+It may sound strange, but some bug reports we receive are completely unrelated to youtube-dl and relate to a different or even the reporter's own application. Please make sure that you are actually using youtube-dl. If you are using a UI for youtube-dl, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for youtube-dl fails in some way you believe is related to youtube-dl, by all means, go ahead and report the bug.
+
+# DEVELOPER INSTRUCTIONS
+
+Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
+
+To run youtube-dl as a developer, you don't need to build anything either. Simply execute
+
+    python -m youtube_dl
+
+To run the test, simply invoke your favorite test runner, or execute a test file directly; any of the following work:
+
+    python -m unittest discover
+    python test/test_download.py
+    nosetests
+
+If you want to create a build of youtube-dl yourself, you'll need
+
+* python
+* make
+* pandoc
+* zip
+* nosetests
+
+### Adding support for a new site
+
+If you want to add support for a new site, you can follow this quick list (assuming your service is called `yourextractor`):
+
+1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
+2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git`
+3. Start a new git branch with `cd youtube-dl; git checkout -b yourextractor`
+4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`:
+    ```python
+    # coding: utf-8
+    from __future__ import unicode_literals
+
+    from .common import InfoExtractor
+
+
+    class YourExtractorIE(InfoExtractor):
+        _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P<id>[0-9]+)'
+        _TEST = {
+            'url': 'http://yourextractor.com/watch/42',
+            'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+            'info_dict': {
+                'id': '42',
+                'ext': 'mp4',
+                'title': 'Video title goes here',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                # TODO more properties, either as:
+                # * A value
+                # * MD5 checksum; start the string with md5:
+                # * A regular expression; start the string with re:
+                # * Any Python type (for example int or float)
+            }
+        }
+
+        def _real_extract(self, url):
+            video_id = self._match_id(url)
+            webpage = self._download_webpage(url, video_id)
+
+            # TODO more code goes here, for example ...
+            title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+
+            return {
+                'id': video_id,
+                'title': title,
+                'description': self._og_search_description(webpage),
+                # TODO more properties (see youtube_dl/extractor/common.py)
+            }
+    ```
+5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
+7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
+8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
+9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
+
+        $ git add youtube_dl/extractor/__init__.py
+        $ git add youtube_dl/extractor/yourextractor.py
+        $ git commit -m '[yourextractor] Add new extractor'
+        $ git push origin yourextractor
+
+10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
+
+In any case, thank you very much for your contributions!
+
+# EMBEDDING YOUTUBE-DL
+
+youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
+
+From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
+
+    import youtube_dl
+
+    ydl_opts = {}
+    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
+        ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+
diff --git a/Makefile b/Makefile
index b846331b9..d5b6e4307 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
-all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
+all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json CONTRIBUTING.md.tmp
 
 cleanall: clean
 	rm -f youtube-dl youtube-dl.exe
@@ -56,6 +56,9 @@ youtube-dl: youtube_dl/*.py youtube_dl/*/*.py
 README.md: youtube_dl/*.py youtube_dl/*/*.py
 	COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py
 
+CONTRIBUTING.md: README.md
+	python devscripts/make_contributing.py README.md CONTRIBUTING.md
+
 README.txt: README.md
 	pandoc -f markdown -t plain README.md -o README.txt
 
diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py
new file mode 100755
index 000000000..99451c568
--- /dev/null
+++ b/devscripts/make_contributing.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import argparse
+import io
+import re
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        'INFILE', help='README.md file name to read from')
+    parser.add_argument(
+        'OUTFILE', help='CONTRIBUTING.md file name to write to')
+    args = parser.parse_args()
+
+    with io.open(args.INFILE, encoding='utf-8') as inf:
+        readme = inf.read()
+
+    bug_text = re.search(
+        r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1)
+    dev_text = re.search(
+        r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*BUGS', readme).group(1)
+
+    out = bug_text + dev_text
+
+    with io.open(args.OUTFILE, 'w', encoding='utf-8') as outf:
+        outf.write(out)
+
+if __name__ == '__main__':
+    main()

From 13dcfd41bda5eaf40637d4054bec33cd804940b9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 16:47:22 +0100
Subject: [PATCH 164/974] [CONTRIBUTING.md] Remove the section about embedding;
 that is not applicable for youtube-dl contributors

---
 CONTRIBUTING.md                 | 14 --------------
 devscripts/make_contributing.py |  3 ++-
 2 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 48840f8f6..0ff7b395a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -134,17 +134,3 @@ If you want to add support for a new site, you can follow this quick list (assum
 
 In any case, thank you very much for your contributions!
 
-# EMBEDDING YOUTUBE-DL
-
-youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
-
-From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
-
-    import youtube_dl
-
-    ydl_opts = {}
-    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
-        ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
-
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
-
diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py
index 99451c568..5fa8cf851 100755
--- a/devscripts/make_contributing.py
+++ b/devscripts/make_contributing.py
@@ -20,7 +20,8 @@ def main():
     bug_text = re.search(
         r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1)
     dev_text = re.search(
-        r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*BUGS', readme).group(1)
+        r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*EMBEDDING YOUTUBE-DL',
+        readme).group(1)
 
     out = bug_text + dev_text
 

From da3a2d8137fa89051b011d90875827504a9a9373 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 16:47:38 +0100
Subject: [PATCH 165/974] release 2014.12.12.3

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 61c106e1a..995bae3d4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.2'
+__version__ = '2014.12.12.3'

From ff815fe65aab83db7e11251db5eafc76504adf05 Mon Sep 17 00:00:00 2001
From: Mark Schreiber <mark7@alumni.cmu.edu>
Date: Thu, 10 Jul 2014 20:11:11 -0700
Subject: [PATCH 166/974] Download playlist items in reverse order

Series of videos are typically uploaded to YouTube playlists in
chronological order.  By default, these videos are downloaded
latest-to-earliest; this is great for seeing the latest videos in a
series, but prevents streaming video in the order that the videos were
produced.  Add an option to download videos in reverse order,
earliest-to-latest.

Conflicts:
	youtube_dl/YoutubeDL.py
	youtube_dl/__init__.py
---
 README.md               | 1 +
 youtube_dl/YoutubeDL.py | 4 ++++
 youtube_dl/__init__.py  | 1 +
 youtube_dl/options.py   | 4 ++++
 4 files changed, 10 insertions(+)

diff --git a/README.md b/README.md
index b3af18912..e41ee8985 100644
--- a/README.md
+++ b/README.md
@@ -75,6 +75,7 @@ which means you can modify it, redistribute it or use it however you like.
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
+    --playlist-reverse               Download playlist videos in reverse order
     --match-title REGEX              download only matching titles (regex or
                                      caseless sub-string)
     --reject-title REGEX             skip download for matching titles (regex or
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e7abb7e0b..578c8daf2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -124,6 +124,7 @@ class YoutubeDL(object):
     nooverwrites:      Prevent overwriting files.
     playliststart:     Playlist item to start at.
     playlistend:       Playlist item to end at.
+    playlistreverse:   Download playlist items in reverse order.
     matchtitle:        Download only matching titles.
     rejecttitle:       Reject downloads for matching titles.
     logger:            Log messages to a logging.Logger instance.
@@ -670,6 +671,9 @@ class YoutubeDL(object):
                     "[%s] playlist %s: Downloading %d videos" %
                     (ie_result['extractor'], playlist, n_entries))
 
+            if self.params.get('playlistreverse', False):
+                entries = entries[::-1]
+
             for i, entry in enumerate(entries, 1):
                 self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries))
                 extra = {
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 77b3384a0..70c4f25b1 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -249,6 +249,7 @@ def _real_main(argv=None):
         'progress_with_newline': opts.progress_with_newline,
         'playliststart': opts.playliststart,
         'playlistend': opts.playlistend,
+        'playlistreverse': opts.playlist_reverse,
         'noplaylist': opts.noplaylist,
         'logtostderr': opts.outtmpl == '-',
         'consoletitle': opts.consoletitle,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 041ca83d8..7654d1a4e 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -349,6 +349,10 @@ def parseOpts(overrideArguments=None):
         '--test',
         action='store_true', dest='test', default=False,
         help=optparse.SUPPRESS_HELP)
+    downloader.add_option(
+        '--playlist-reverse',
+        action='store_true',
+        help='Download playlist videos in reverse order')
 
     workarounds = optparse.OptionGroup(parser, 'Workarounds')
     workarounds.add_option(

From 8bdcb436f9bf90b90ddd9b93defff644760ebf02 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:06:52 +0100
Subject: [PATCH 167/974] [test_unicode_literals] Fix test

---
 test/helper.py                |  4 +++-
 test/test_unicode_literals.py | 14 ++++++++++++--
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 9a7f0746e..8a820526a 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -161,7 +161,9 @@ def assertRegexpMatches(self, text, regexp, msg=None):
     else:
         m = re.match(regexp, text)
         if not m:
-            note = 'Regexp didn\'t match: %r not found in %r' % (regexp, text)
+            note = 'Regexp didn\'t match: %r not found' % (regexp)
+            if len(text) < 1000:
+                note += ' in %r' % text
             if msg is None:
                 msg = note
             else:
diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index 2cc431b0b..d3cba869b 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -1,5 +1,11 @@
 from __future__ import unicode_literals
 
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 import io
 import os
 import re
@@ -14,6 +20,9 @@ IGNORED_FILES = [
 ]
 
 
+from helper import assertRegexpMatches
+
+
 class TestUnicodeLiterals(unittest.TestCase):
     def test_all_files(self):
         for dirpath, _, filenames in os.walk(rootDir):
@@ -29,9 +38,10 @@ class TestUnicodeLiterals(unittest.TestCase):
 
                 if "'" not in code and '"' not in code:
                     continue
-                self.assertRegexpMatches(
+                assertRegexpMatches(
+                    self,
                     code,
-                    r'(?:#.*\n*)?from __future__ import (?:[a-z_]+,\s*)*unicode_literals',
+                    r'(?:(?:#.*?|\s*)\n)*from __future__ import (?:[a-z_]+,\s*)*unicode_literals',
                     'unicode_literals import  missing in %s' % fn)
 
                 m = re.search(r'(?<=\s)u[\'"](?!\)|,|$)', code)

From 7c7a0d395c2eb60beb6ddf2fab19b89cae6a2cdc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:07:39 +0100
Subject: [PATCH 168/974] Remove unused imports

---
 youtube_dl/extractor/cnet.py            | 2 --
 youtube_dl/extractor/goldenmoustache.py | 3 ---
 2 files changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index a6f1f3a91..3145b3051 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -2,12 +2,10 @@
 from __future__ import unicode_literals
 
 import json
-import re
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    int_or_none,
 )
 
 
diff --git a/youtube_dl/extractor/goldenmoustache.py b/youtube_dl/extractor/goldenmoustache.py
index 4a07b5558..0fb509724 100644
--- a/youtube_dl/extractor/goldenmoustache.py
+++ b/youtube_dl/extractor/goldenmoustache.py
@@ -1,9 +1,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-)
 
 
 class GoldenMoustacheIE(InfoExtractor):

From 58c1f6f0a708eb2b084e773d7aa74bfe37d80c6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Dec 2014 22:10:32 +0600
Subject: [PATCH 169/974] [nbc] Fix extraction (Closes #4441)

---
 youtube_dl/extractor/nbc.py | 36 +++++++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 7b5449031..bf5132721 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -12,23 +12,37 @@ from ..utils import (
 
 
 class NBCIE(InfoExtractor):
-    _VALID_URL = r'http://www\.nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+)'
+    _VALID_URL = r'http://www\.nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'
 
-    _TEST = {
-        'url': 'http://www.nbc.com/chicago-fire/video/i-am-a-firefighter/2734188',
-        # md5 checksum is not stable
-        'info_dict': {
-            'id': 'bTmnLCvIbaaH',
-            'ext': 'flv',
-            'title': 'I Am a Firefighter',
-            'description': 'An emergency puts Dawson\'sf irefighter skills to the ultimate test in this four-part digital series.',
+    _TESTS = [
+        {
+            'url': 'http://www.nbc.com/chicago-fire/video/i-am-a-firefighter/2734188',
+            # md5 checksum is not stable
+            'info_dict': {
+                'id': 'bTmnLCvIbaaH',
+                'ext': 'flv',
+                'title': 'I Am a Firefighter',
+                'description': 'An emergency puts Dawson\'sf irefighter skills to the ultimate test in this four-part digital series.',
+            },
         },
-    }
+        {
+            'url': 'http://www.nbc.com/the-tonight-show/episodes/176',
+            'info_dict': {
+                'id': 'XwU9KZkp98TH',
+                'ext': 'flv',
+                'title': 'Ricky Gervais, Steven Van Zandt, ILoveMakonnen',
+                'description': 'A brand new episode of The Tonight Show welcomes Ricky Gervais, Steven Van Zandt and ILoveMakonnen.',
+            },
+            'skip': 'Only works from US',
+        },
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        theplatform_url = self._search_regex('class="video-player video-player-full" data-mpx-url="(.*?)"', webpage, 'theplatform url')
+        theplatform_url = self._search_regex(
+            '(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
+            webpage, 'theplatform url').replace('_no_endcard', '')
         if theplatform_url.startswith('//'):
             theplatform_url = 'http:' + theplatform_url
         return self.url_result(theplatform_url)

From 2865cf0419a4c92133053f128934c64df0dccda9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:11:53 +0100
Subject: [PATCH 170/974] Deprecate --auto-number (Closes #2704)

---
 youtube_dl/options.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 7654d1a4e..21c452141 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -484,10 +484,6 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '--id', default=False,
         action='store_true', dest='useid', help='use only video ID in file name')
-    filesystem.add_option(
-        '-A', '--auto-number',
-        action='store_true', dest='autonumber', default=False,
-        help='number downloaded files starting from 00000')
     filesystem.add_option(
         '-o', '--output',
         dest='outtmpl', metavar='TEMPLATE',
@@ -515,6 +511,10 @@ def parseOpts(overrideArguments=None):
         '--restrict-filenames',
         action='store_true', dest='restrictfilenames', default=False,
         help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames')
+    filesystem.add_option(
+        '-A', '--auto-number',
+        action='store_true', dest='autonumber', default=False,
+        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] number downloaded files starting from 00000')
     filesystem.add_option(
         '-t', '--title',
         action='store_true', dest='usetitle', default=False,

From cc8c9281e6313c8b108f20df0ea633445c78bf2d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:17:09 +0100
Subject: [PATCH 171/974] [downloader/common] Do not use classic int division

---
 youtube_dl/downloader/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 6e44c88c9..2a566eabe 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -160,7 +160,7 @@ class FileDownloader(object):
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:
-            time.sleep(max((byte_counter / rate_limit) - elapsed, 0))
+            time.sleep(max((byte_counter // rate_limit) - elapsed, 0))
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""

From 2101f5d4ccd5e568a2b860596e67d01364b9078b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:18:22 +0100
Subject: [PATCH 172/974] release 2014.12.12.4

---
 README.md             | 6 ++++--
 youtube_dl/version.py | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index e41ee8985..f10f06ee8 100644
--- a/README.md
+++ b/README.md
@@ -75,7 +75,6 @@ which means you can modify it, redistribute it or use it however you like.
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
-    --playlist-reverse               Download playlist videos in reverse order
     --match-title REGEX              download only matching titles (regex or
                                      caseless sub-string)
     --reject-title REGEX             skip download for matching titles (regex or
@@ -114,12 +113,12 @@ which means you can modify it, redistribute it or use it however you like.
                                      size. By default, the buffer size is
                                      automatically resized from an initial value
                                      of SIZE.
+    --playlist-reverse               Download playlist videos in reverse order
 
 ## Filesystem Options:
     -a, --batch-file FILE            file containing URLs to download ('-' for
                                      stdin)
     --id                             use only video ID in file name
-    -A, --auto-number                number downloaded files starting from 00000
     -o, --output TEMPLATE            output filename template. Use %(title)s to
                                      get the title, %(uploader)s for the
                                      uploader name, %(uploader_id)s for the
@@ -153,6 +152,9 @@ which means you can modify it, redistribute it or use it however you like.
     --restrict-filenames             Restrict filenames to only ASCII
                                      characters, and avoid "&" and spaces in
                                      filenames
+    -A, --auto-number                [deprecated; use  -o
+                                     "%(autonumber)s-%(title)s.%(ext)s" ] number
+                                     downloaded files starting from 00000
     -t, --title                      [deprecated] use title in file name
                                      (default)
     -l, --literal                    [deprecated] alias of --title
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 995bae3d4..65290b43d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.3'
+__version__ = '2014.12.12.4'

From 57e086dcea6a9981c4943b6a281c0c7f806efaf8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:24:05 +0100
Subject: [PATCH 173/974] [ebaumsworld] Modernize

---
 youtube_dl/extractor/ebaumsworld.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ebaumsworld.py b/youtube_dl/extractor/ebaumsworld.py
index 63c2549d3..b6bfd2b2d 100644
--- a/youtube_dl/extractor/ebaumsworld.py
+++ b/youtube_dl/extractor/ebaumsworld.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -20,8 +18,7 @@ class EbaumsWorldIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         config = self._download_xml(
             'http://www.ebaumsworld.com/video/player/%s' % video_id, video_id)
         video_url = config.find('file').text

From 146323a7f8bf434de91298e48a81a8e0ff689244 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:39:33 +0100
Subject: [PATCH 174/974] [groupon] Add extractor (Fixes #4386)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/groupon.py  | 50 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/groupon.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6866f06d4..104dce1eb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -159,6 +159,7 @@ from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
 from .goshgay import GoshgayIE
 from .grooveshark import GroovesharkIE
+from .groupon import GrouponIE
 from .hark import HarkIE
 from .heise import HeiseIE
 from .helsinki import HelsinkiIE
diff --git a/youtube_dl/extractor/groupon.py b/youtube_dl/extractor/groupon.py
new file mode 100644
index 000000000..8b9e0e2f8
--- /dev/null
+++ b/youtube_dl/extractor/groupon.py
@@ -0,0 +1,50 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class GrouponIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.groupon\.com/deals/(?P<id>[^?#]+)'
+
+    _TEST = {
+        'url': 'https://www.groupon.com/deals/bikram-yoga-huntington-beach-2#ooid=tubGNycTo_9Uxg82uESj4i61EYX8nyuf',
+        'info_dict': {
+            'id': 'bikram-yoga-huntington-beach-2',
+            'title': '$49 for 10 Yoga Classes or One Month of Unlimited Classes at Bikram Yoga Huntington Beach ($180 Value)',
+            'description': 'Studio kept at 105 degrees and 40% humidity with anti-microbial and anti-slip Flotex flooring; certified instructors',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': 'tubGNycTo_9Uxg82uESj4i61EYX8nyuf',
+                'ext': 'mp4',
+                'title': 'Bikram Yoga Huntington Beach | Orange County',
+            },
+        }],
+        'params': {
+            'skip_download': 'HLS',
+        }
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+
+        payload = self._parse_json(self._search_regex(
+            r'var\s+payload\s*=\s*(.*?);\n', webpage, 'payload'), playlist_id)
+        videos = payload['carousel'].get('dealVideos', [])
+        entries = []
+        for v in videos:
+            if v.get('provider') != 'OOYALA':
+                self.report_warning(
+                    '%s: Unsupported video provider %s, skipping video' %
+                    (playlist_id, v.get('provider')))
+                continue
+            entries.append(self.url_result('ooyala:%s' % v['media']))
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'entries': entries,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+        }

From a099965badd87efb31c18603de6d350bbf5b1a24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:40:27 +0100
Subject: [PATCH 175/974] release 2014.12.12.5

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 65290b43d..eea06b9d1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.4'
+__version__ = '2014.12.12.5'

From 07bc9a35308f2ac3ae21de5c53380bc4d01aa72c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Dec 2014 22:44:16 +0600
Subject: [PATCH 176/974] [nowvideo] Add .li domain (Closes #4453)

---
 youtube_dl/extractor/nowvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowvideo.py b/youtube_dl/extractor/nowvideo.py
index ecb38de2d..dec09cdfe 100644
--- a/youtube_dl/extractor/nowvideo.py
+++ b/youtube_dl/extractor/nowvideo.py
@@ -7,7 +7,7 @@ class NowVideoIE(NovaMovIE):
     IE_NAME = 'nowvideo'
     IE_DESC = 'NowVideo'
 
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|sx|eu|at|ag|co)'}
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|sx|eu|at|ag|co|li)'}
 
     _HOST = 'www.nowvideo.ch'
 

From 7a60322abf5bfc72b8a6de62f031b1b3ecb52847 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:52:50 +0100
Subject: [PATCH 177/974] release 2014.12.12.6

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index eea06b9d1..24d1c08f6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.5'
+__version__ = '2014.12.12.6'

From 1c985da0cabfa4b9e93617f743185c4123ea07bb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 18:25:58 +0100
Subject: [PATCH 178/974] release 2014.12.12.7

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 24d1c08f6..da959ea85 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.6'
+__version__ = '2014.12.12.7'

From c3f3b29b92dd46524dcbf9f4b0d957eaa0d0db0c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 12 Dec 2014 20:22:24 +0200
Subject: [PATCH 179/974] [rtp] Add new extractor (Closes #4382)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rtp.py      | 57 ++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 youtube_dl/extractor/rtp.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 104dce1eb..447833c01 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -323,6 +323,7 @@ from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
 from .rtlnl import RtlXlIE
 from .rtlnow import RTLnowIE
+from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE
 from .ruhd import RUHDIE
diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
new file mode 100644
index 000000000..0bed1f100
--- /dev/null
+++ b/youtube_dl/extractor/rtp.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import js_to_json
+
+
+class RTPIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/e(?P<id>[0-9]+)/?'
+    _TEST = {
+        'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
+        'info_dict': {
+            'id': '174042',
+            'ext': 'mp3',
+            'title': 'Paixões Cruzadas',
+            'description': 'As paixões musicais de António Cartaxo e António Macedo',
+            'thumbnail': 're:^https?://.*\.jpg',
+        },
+        'params': {
+            'skip_download': True,  # RTMP download
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_meta(
+            'twitter:title', webpage, display_name='title', fatal=True)
+        description = self._html_search_meta('description', webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        player_config = self._search_regex(
+            r'(?s)RTPPLAY\.player\.newPlayer\(\s*(\{.*?\})\s*\)', webpage, 'player config')
+        config = json.loads(js_to_json(player_config))
+
+        path, ext = config.get('file').rsplit('.', 1)
+        formats = [{
+            'app': config.get('application'),
+            'play_path': '{ext:s}:{path:s}'.format(ext=ext, path=path),
+            'page_url': url,
+            'url': 'rtmp://{streamer:s}/{application:s}'.format(**config),
+            'rtmp_live': config.get('live', False),
+            'ext': ext,
+            'vcodec': config.get('type') == 'audio' and 'none' or None,
+            'player_url': 'http://programas.rtp.pt/play/player.swf?v3',
+        }]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'thumbnail': thumbnail,
+        }

From 4237ba10dc46ffd0c95fbe2d077521da98726fe4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 19:44:25 +0100
Subject: [PATCH 180/974] [pornotube] Adapt to new interface

---
 youtube_dl/extractor/pornotube.py | 102 ++++++++++++++++++++----------
 1 file changed, 70 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/pornotube.py b/youtube_dl/extractor/pornotube.py
index 5253aa3d3..34735c51e 100644
--- a/youtube_dl/extractor/pornotube.py
+++ b/youtube_dl/extractor/pornotube.py
@@ -1,56 +1,94 @@
 from __future__ import unicode_literals
 
-import re
+import json
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+)
 from ..utils import (
-    compat_urllib_parse,
-
-    unified_strdate,
+    int_or_none,
 )
 
 
 class PornotubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$'
+    _VALID_URL = r'https?://(?:\w+\.)?pornotube\.com/(?:[^?#]*?)/video/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing',
-        'md5': '374dd6dcedd24234453b295209aa69b6',
+        'url': 'http://www.pornotube.com/orientation/straight/video/4964/title/weird-hot-and-wet-science',
+        'md5': '60fc5a4f0d93a97968fc7999d98260c9',
         'info_dict': {
-            'id': '1689755',
-            'ext': 'flv',
-            'upload_date': '20090708',
-            'title': 'Marilyn-Monroe-Bathing',
-            'age_limit': 18
+            'id': '4964',
+            'ext': 'mp4',
+            'upload_date': '20141203',
+            'title': 'Weird Hot and Wet Science',
+            'description': 'md5:a8304bef7ef06cb4ab476ca6029b01b0',
+            'categories': ['Adult Humor', 'Blondes'],
+            'uploader': 'Alpha Blue Archives',
+            'thumbnail': 're:^https?://.*\\.jpg$',
+            'timestamp': 1417582800,
+            'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        video_id = self._match_id(url)
 
-        video_id = mobj.group('videoid')
-        video_title = mobj.group('title')
+        # Fetch origin token
+        js_config = self._download_webpage(
+            'http://www.pornotube.com/assets/src/app/config.js', video_id,
+            note='Download JS config')
+        originAuthenticationSpaceKey = self._search_regex(
+            r"constant\('originAuthenticationSpaceKey',\s*'([^']+)'",
+            js_config, 'originAuthenticationSpaceKey')
 
-        # Get webpage content
-        webpage = self._download_webpage(url, video_id)
+        # Fetch actual token
+        token_req_data = {
+            'authenticationSpaceKey': originAuthenticationSpaceKey,
+            'credentials': 'Clip Application',
+        }
+        token_req = compat_urllib_request.Request(
+            'https://api.aebn.net/auth/v1/token/primal',
+            data=json.dumps(token_req_data).encode('utf-8'))
+        token_req.add_header('Content-Type', 'application/json')
+        token_req.add_header('Origin', 'http://www.pornotube.com')
+        token_answer = self._download_json(
+            token_req, video_id, note='Requesting primal token')
+        token = token_answer['tokenKey']
 
-        # Get the video URL
-        VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",'
-        video_url = self._search_regex(VIDEO_URL_RE, webpage, 'video url')
-        video_url = compat_urllib_parse.unquote(video_url)
+        # Get video URL
+        delivery_req = compat_urllib_request.Request(
+            'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id)
+        delivery_req.add_header('Authorization', token)
+        delivery_info = self._download_json(
+            delivery_req, video_id, note='Downloading delivery information')
+        video_url = delivery_info['mediaUrl']
 
-        # Get the uploaded date
-        VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by'
-        upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, 'upload date', fatal=False)
-        if upload_date:
-            upload_date = unified_strdate(upload_date)
-        age_limit = self._rta_search(webpage)
+        # Get additional info (title etc.)
+        info_req = compat_urllib_request.Request(
+            'https://api.aebn.net/content/v1/clips/%s?expand='
+            'title,description,primaryImageNumber,startSecond,endSecond,'
+            'movie.title,movie.MovieId,movie.boxCoverFront,movie.stars,'
+            'movie.studios,stars.name,studios.name,categories.name,'
+            'clipActive,movieActive,publishDate,orientations' % video_id)
+        info_req.add_header('Authorization', token)
+        info = self._download_json(
+            info_req, video_id, note='Downloading metadata')
+
+        timestamp = int_or_none(info.get('publishDate'), scale=1000)
+        uploader = info.get('studios', [{}])[0].get('name')
+        movie_id = info['movie']['movieId']
+        thumbnail = 'http://pic.aebn.net/dis/t/%s/%s_%08d.jpg' % (
+            movie_id, movie_id, info['primaryImageNumber'])
+        categories = [c['name'] for c in info.get('categories')]
 
         return {
             'id': video_id,
             'url': video_url,
-            'upload_date': upload_date,
-            'title': video_title,
-            'ext': 'flv',
-            'format': 'flv',
-            'age_limit': age_limit,
+            'title': info['title'],
+            'description': info.get('description'),
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'age_limit': 18,
         }

From dc65a213fd73a385470fe1d8f7ad48a174d11bd4 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Fri, 12 Dec 2014 19:58:44 +0100
Subject: [PATCH 181/974] comediansincarsgettingcoffee.com support

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ccc.py      | 46 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/ccc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 447833c01..bd40e9ea5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -48,6 +48,7 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
+from .ccc import ComCarCoffIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
new file mode 100644
index 000000000..809725f12
--- /dev/null
+++ b/youtube_dl/extractor/ccc.py
@@ -0,0 +1,46 @@
+# encoding: utf-8
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+)
+
+class ComCarCoffIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]+)/?'
+    _TESTS = [
+        {
+            'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
+            'info_dict': {
+                'id': 'miranda-sings-happy-thanksgiving-miranda',
+                'upload_date': '20141127',
+                'title': 'Happy Thanksgiving Miranda',
+                'description': 'Jerry Seinfeld and his special guest Miranda Sings cruise around town in search of coffee, complaining and apologizing along the way.',
+                'thumbnail': 'http://ccc.crackle.com/images/s5e4_thumb.jpg',
+            },
+        }
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        full_data = json.loads(self._search_regex(
+            r'<script type="application/json" id="videoData">(?P<json>.+?)</script>',
+            webpage, 'json'))
+
+        video_id = full_data['activeVideo']['video']
+        video_data = full_data['videos'][video_id]
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': video_data['title'],
+            'description': video_data['description'],
+            # XXX: the original datum is a full ISO timestamp... why convert it to a worse format?
+            'upload_date': unified_strdate(video_data['pubDate']),
+            'thumbnail': video_data['images']['thumb'],
+            # XXX: what do we do with video_data['images']['poster']?
+            'formats': self._extract_m3u8_formats(video_data['mediaUrl'], video_id),
+        }

From e9404524cc108d2a054a9aaeaecc0ac88f451676 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:10:15 +0100
Subject: [PATCH 182/974] [ninegag] Test for additional properties

---
 youtube_dl/extractor/ninegag.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 16a02ad79..7f842b5c2 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -23,6 +23,9 @@ class NineGagIE(InfoExtractor):
             "ext": "mp4",
             "description": "This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)",
             "title": "\"People Are Awesome 2013\" Is Absolutely Awesome",
+            'uploader_id': 'UCdEH6EjDKwtTe-sO2f0_1XA',
+            'uploader': 'CompilationChannel',
+            'upload_date': '20131110',
             "view_count": int,
             "thumbnail": "re:^https?://",
         },
@@ -35,6 +38,9 @@ class NineGagIE(InfoExtractor):
             'display_id': 'alternate-banned-opening-scene-of-gravity',
             "description": "While Gravity was a pretty awesome movie already, YouTuber Krishna Shenoi came up with a way to improve upon it, introducing a much better solution to Sandra Bullock's seemingly endless tumble in space. The ending is priceless.",
             'title': "Banned Opening Scene Of \"Gravity\" That Changes The Whole Movie",
+            'uploader': 'Krishna Shenoi',
+            'upload_date': '20140401',
+            'uploader_id': 'krishnashenoi93',
         },
     }]
 

From 90d71d3f088bbb787795aa6018eda21e2a821e7d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:11:23 +0100
Subject: [PATCH 183/974] [ooyala] Remove test md5sums

---
 youtube_dl/extractor/ooyala.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index f17a52858..d5b05c18f 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -16,7 +16,6 @@ class OoyalaIE(InfoExtractor):
         {
             # From http://it.slashdot.org/story/13/04/25/178216/recovering-data-from-broken-hard-drives-and-ssds-video
             'url': 'http://player.ooyala.com/player.js?embedCode=pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
-            'md5': '3f5cceb3a7bf461d6c29dc466cf8033c',
             'info_dict': {
                 'id': 'pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
                 'ext': 'mp4',
@@ -26,7 +25,6 @@ class OoyalaIE(InfoExtractor):
         }, {
             # Only available for ipad
             'url': 'http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
-            'md5': '4b9754921fddb68106e48c142e2a01e6',
             'info_dict': {
                 'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
                 'ext': 'mp4',

From dc5596ff54860dc470579c17f4b072266f44cf5c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:32:02 +0100
Subject: [PATCH 184/974] [comcarcoff] (#4454)

---
 youtube_dl/extractor/__init__.py   |  2 +-
 youtube_dl/extractor/ccc.py        | 46 --------------------------
 youtube_dl/extractor/comcarcoff.py | 52 ++++++++++++++++++++++++++++++
 3 files changed, 53 insertions(+), 47 deletions(-)
 delete mode 100644 youtube_dl/extractor/ccc.py
 create mode 100644 youtube_dl/extractor/comcarcoff.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd40e9ea5..119ec2044 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -48,7 +48,6 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
-from .ccc import ComCarCoffIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
@@ -66,6 +65,7 @@ from .cnn import (
 )
 from .collegehumor import CollegeHumorIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
+from .comcarcoff import ComCarCoffIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
deleted file mode 100644
index 809725f12..000000000
--- a/youtube_dl/extractor/ccc.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# encoding: utf-8
-import re
-import json
-
-from .common import InfoExtractor
-from ..utils import (
-    unified_strdate,
-)
-
-class ComCarCoffIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]+)/?'
-    _TESTS = [
-        {
-            'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
-            'info_dict': {
-                'id': 'miranda-sings-happy-thanksgiving-miranda',
-                'upload_date': '20141127',
-                'title': 'Happy Thanksgiving Miranda',
-                'description': 'Jerry Seinfeld and his special guest Miranda Sings cruise around town in search of coffee, complaining and apologizing along the way.',
-                'thumbnail': 'http://ccc.crackle.com/images/s5e4_thumb.jpg',
-            },
-        }
-    ]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        full_data = json.loads(self._search_regex(
-            r'<script type="application/json" id="videoData">(?P<json>.+?)</script>',
-            webpage, 'json'))
-
-        video_id = full_data['activeVideo']['video']
-        video_data = full_data['videos'][video_id]
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': video_data['title'],
-            'description': video_data['description'],
-            # XXX: the original datum is a full ISO timestamp... why convert it to a worse format?
-            'upload_date': unified_strdate(video_data['pubDate']),
-            'thumbnail': video_data['images']['thumb'],
-            # XXX: what do we do with video_data['images']['poster']?
-            'formats': self._extract_m3u8_formats(video_data['mediaUrl'], video_id),
-        }
diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
new file mode 100644
index 000000000..46953c12e
--- /dev/null
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -0,0 +1,52 @@
+# encoding: utf-8
+import json
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601
+
+
+class ComCarCoffIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]+)/?'
+    _TESTS = [{
+        'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
+        'info_dict': {
+            'id': 'miranda-sings-happy-thanksgiving-miranda',
+            'ext': 'mp4',
+            'upload_date': '20141127',
+            'timestamp': 1417107600,
+            'title': 'Happy Thanksgiving Miranda',
+            'description': 'Jerry Seinfeld and his special guest Miranda Sings cruise around town in search of coffee, complaining and apologizing along the way.',
+            'thumbnail': 'http://ccc.crackle.com/images/s5e4_thumb.jpg',
+        },
+        'params': {
+            'skip_download': 'requires ffmpeg',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        full_data = json.loads(self._search_regex(
+            r'<script type="application/json" id="videoData">(?P<json>.+?)</script>',
+            webpage, 'full data json'))
+
+        video_id = full_data['activeVideo']['video']
+        video_data = full_data['videos'][video_id]
+        thumbnails = [{
+            'url': video_data['images']['thumb'],
+        }, {
+            'url': video_data['images']['poster'],
+        }]
+        formats = self._extract_m3u8_formats(
+            video_data['mediaUrl'], video_id, ext='mp4')
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': video_data['title'],
+            'description': video_data.get('description'),
+            'timestamp': parse_iso8601(video_data.get('pubDate')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

From 3c864e930dd7533b404d03872d078e038883ac3d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:35:17 +0100
Subject: [PATCH 185/974] [comcarcoff] Adapt
 c62159ea91a04ef82560472b254aef1cc9f70a11

---
 youtube_dl/extractor/comcarcoff.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 46953c12e..67724139c 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -6,7 +6,7 @@ from ..utils import parse_iso8601
 
 
 class ComCarCoffIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]+)/?'
+    _VALID_URL = r'http://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]*)'
     _TESTS = [{
         'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
         'info_dict': {
@@ -25,6 +25,8 @@ class ComCarCoffIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+        if not display_id:
+            display_id = 'comediansincarsgettingcoffee.com'
         webpage = self._download_webpage(url, display_id)
 
         full_data = json.loads(self._search_regex(

From 6a1b20de2a562f76501ae96721957f43804d8e61 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:37:28 +0100
Subject: [PATCH 186/974] [urort] Modernize

---
 youtube_dl/extractor/urort.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/urort.py b/youtube_dl/extractor/urort.py
index 5d06fcc9e..468aa7600 100644
--- a/youtube_dl/extractor/urort.py
+++ b/youtube_dl/extractor/urort.py
@@ -33,13 +33,11 @@ class UrortIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
+        playlist_id = self._match_id(url)
 
         fstr = compat_urllib_parse.quote("InternalBandUrl eq '%s'" % playlist_id)
         json_url = 'http://urort.p3.no/breeze/urort/TrackDtos?$filter=' + fstr
         songs = self._download_json(json_url, playlist_id)
-        print(songs[0])
 
         entries = [{
             'id': '%d-%s' % (s['BandId'], s['$id']),

From 19bf2b4e88a1d71f5d617a6e9728a9c2687a8df8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:37:58 +0100
Subject: [PATCH 187/974] [comcarcoff] Add unicode_literals declaration

---
 youtube_dl/extractor/comcarcoff.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 67724139c..a2211ee44 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -1,4 +1,6 @@
 # encoding: utf-8
+from __future__ import unicode_literals
+
 import json
 
 from .common import InfoExtractor

From baa7081d68996377e44225c74a1ec05e801617a2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:55:18 +0100
Subject: [PATCH 188/974] [urort] Update to new multi-format protocol

---
 youtube_dl/extractor/urort.py | 37 +++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/urort.py b/youtube_dl/extractor/urort.py
index 468aa7600..249a34c86 100644
--- a/youtube_dl/extractor/urort.py
+++ b/youtube_dl/extractor/urort.py
@@ -18,11 +18,10 @@ class UrortIE(InfoExtractor):
         'url': 'https://urort.p3.no/#!/Band/Gerilja',
         'md5': '5ed31a924be8a05e47812678a86e127b',
         'info_dict': {
-            'id': '33124-4',
+            'id': '33124-24',
             'ext': 'mp3',
             'title': 'The Bomb',
             'thumbnail': 're:^https?://.+\.jpg',
-            'like_count': int,
             'uploader': 'Gerilja',
             'uploader_id': 'Gerilja',
             'upload_date': '20100323',
@@ -36,20 +35,28 @@ class UrortIE(InfoExtractor):
         playlist_id = self._match_id(url)
 
         fstr = compat_urllib_parse.quote("InternalBandUrl eq '%s'" % playlist_id)
-        json_url = 'http://urort.p3.no/breeze/urort/TrackDtos?$filter=' + fstr
+        json_url = 'http://urort.p3.no/breeze/urort/TrackDTOViews?$filter=%s&$orderby=Released%%20desc&$expand=Tags%%2CFiles' % fstr
         songs = self._download_json(json_url, playlist_id)
-
-        entries = [{
-            'id': '%d-%s' % (s['BandId'], s['$id']),
-            'title': s['Title'],
-            'url': s['TrackUrl'],
-            'ext': 'mp3',
-            'uploader_id': playlist_id,
-            'uploader': s.get('BandName', playlist_id),
-            'like_count': s.get('LikeCount'),
-            'thumbnail': 'http://urort.p3.no/cloud/images/%s' % s['Image'],
-            'upload_date': unified_strdate(s.get('Released')),
-        } for s in songs]
+        entries = []
+        for s in songs:
+            formats = [{
+                'tbr': f.get('Quality'),
+                'ext': f['FileType'],
+                'format_id': '%s-%s' % (f['FileType'], f.get('Quality', '')),
+                'url': 'http://p3urort.blob.core.windows.net/tracks/%s' % f['FileRef'],
+                'preference': 3 if f['FileType'] == 'mp3' else 2,
+            } for f in s['Files']]
+            self._sort_formats(formats)
+            e = {
+                'id': '%d-%s' % (s['BandId'], s['$id']),
+                'title': s['Title'],
+                'uploader_id': playlist_id,
+                'uploader': s.get('BandName', playlist_id),
+                'thumbnail': 'http://urort.p3.no/cloud/images/%s' % s['Image'],
+                'upload_date': unified_strdate(s.get('Released')),
+                'formats': formats,
+            }
+            entries.append(e)
 
         return {
             '_type': 'playlist',

From 23d3608c6b5b50cbbc81314d18824c4951f8af27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 12 Dec 2014 22:23:54 +0100
Subject: [PATCH 189/974] [youtube:channel] Fix extraction (fixes #4435)

It uses now the same pagination system as playlists
---
 youtube_dl/extractor/youtube.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7b6179a2a..a7bdce72f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1269,8 +1269,6 @@ class YoutubeTopListIE(YoutubePlaylistIE):
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = 'YouTube.com channels'
     _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
-    _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
-    _MORE_PAGES_URL = 'https://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
     IE_NAME = 'youtube:channel'
     _TESTS = [{
         'note': 'paginated channel',
@@ -1307,20 +1305,27 @@ class YoutubeChannelIE(InfoExtractor):
             return self.playlist_result(entries, channel_id)
 
         def _entries():
+            more_widget_html = content_html = channel_page
             for pagenum in itertools.count(1):
-                url = self._MORE_PAGES_URL % (pagenum, channel_id)
-                page = self._download_json(
-                    url, channel_id, note='Downloading page #%s' % pagenum,
-                    transform_source=uppercase_escape)
 
-                ids_in_page = self.extract_videos_from_page(page['content_html'])
+                ids_in_page = self.extract_videos_from_page(content_html)
                 for video_id in ids_in_page:
                     yield self.url_result(
                         video_id, 'Youtube', video_id=video_id)
 
-                if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
+                mobj = re.search(
+                    r'data-uix-load-more-href="/?(?P<more>[^"]+)"',
+                    more_widget_html)
+                if not mobj:
                     break
 
+                more = self._download_json(
+                    'https://youtube.com/%s' % mobj.group('more'), channel_id,
+                    'Downloading page #%s' % (pagenum + 1),
+                    transform_source=uppercase_escape)
+                content_html = more['content_html']
+                more_widget_html = more['load_more_widget_html']
+
         return self.playlist_result(_entries(), channel_id)
 
 

From 26c06f0c513347edae6b1005b875a634ec386d19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 12 Dec 2014 22:26:50 +0100
Subject: [PATCH 190/974] [youtube:playlist] Remove unused property

---
 youtube_dl/extractor/youtube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a7bdce72f..823d6aaf3 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1072,7 +1072,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                         ((?:PL|LL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,})
                      )"""
     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
-    _MORE_PAGES_INDICATOR = r'data-link-type="next"'
     _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)'
     IE_NAME = 'youtube:playlist'
     _TESTS = [{

From 7668a2c5cba305bf7dc41b14149fa7f55d4dffd6 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Fri, 12 Dec 2014 20:49:50 +0100
Subject: [PATCH 191/974] [comcarcoff] add webpage_url datum

---
 youtube_dl/extractor/comcarcoff.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 67724139c..b62be56da 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -51,4 +51,5 @@ class ComCarCoffIE(InfoExtractor):
             'timestamp': parse_iso8601(video_data.get('pubDate')),
             'thumbnails': thumbnails,
             'formats': formats,
+            'webpage_url': 'http://comediansincarsgettingcoffee.com/%s' % (video_data.get('urlSlug', video_data.get('slug'))),
         }

From 0db261ba567cb5370455d67c4398e11e5e2119f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 12 Dec 2014 23:27:25 +0100
Subject: [PATCH 192/974] [utils] make_HTTPS_handler:  Use
 ssl.create_default_context in Python 2.7.9

The new features in the ssl module have been backported from 3.4, see https://docs.python.org/dev/whatsnew/2.7.html#pep-466-network-security-enhancements-for-python-2-7
---
 youtube_dl/utils.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 51a822e4f..83a45e702 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -388,7 +388,13 @@ def formatSeconds(secs):
 
 
 def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
-    if sys.version_info < (3, 2):
+    if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
+        context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+        context.options &= ~ssl.OP_NO_SSLv3  # Allow older, not-as-secure SSLv3
+        if opts_no_check_certificate:
+            context.verify_mode = ssl.CERT_NONE
+        return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+    elif sys.version_info < (3, 2):
         import httplib
 
         class HTTPSConnectionV3(httplib.HTTPSConnection):
@@ -409,12 +415,6 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
             def https_open(self, req):
                 return self.do_open(HTTPSConnectionV3, req)
         return HTTPSHandlerV3(**kwargs)
-    elif hasattr(ssl, 'create_default_context'):  # Python >= 3.4
-        context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
-        context.options &= ~ssl.OP_NO_SSLv3  # Allow older, not-as-secure SSLv3
-        if opts_no_check_certificate:
-            context.verify_mode = ssl.CERT_NONE
-        return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
     else:  # Python < 3.4
         context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
         context.verify_mode = (ssl.CERT_NONE

From 293f0f39ce955b9aa8284d461a0444e27491c392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 12 Dec 2014 23:43:25 +0100
Subject: [PATCH 193/974] [utils] make_HTTPS_handler: Remove try/except block
 that would always raise an exception

This code is only run for Python < 3.4, where context.load_default_certs doesn't exist
---
 youtube_dl/utils.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 83a45e702..ec34dcef9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -421,10 +421,6 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
                                if opts_no_check_certificate
                                else ssl.CERT_REQUIRED)
         context.set_default_verify_paths()
-        try:
-            context.load_default_certs()
-        except AttributeError:
-            pass  # Python < 3.4
         return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
 
 

From 5e1912cfc102f457fb9cb2472fb93c973cb68732 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 11:02:24 +0100
Subject: [PATCH 194/974] [5min] Remove helper method and modernize

Previously, other extractor would go call a private(!) helper method. Instead, just hardcode the 5min:video_id format - it's not if that would ever change.
---
 youtube_dl/extractor/aol.py      | 49 ++++++++++++++++----------------
 youtube_dl/extractor/engadget.py |  8 ++----
 youtube_dl/extractor/fivemin.py  | 15 ++++------
 3 files changed, 32 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index 47f8e4157..185ee3693 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .fivemin import FiveMinIE
 
 
 class AolIE(InfoExtractor):
@@ -42,31 +41,31 @@ class AolIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-
         playlist_id = mobj.group('playlist_id')
-        if playlist_id and not self._downloader.params.get('noplaylist'):
-            self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
+        if not playlist_id or self._downloader.params.get('noplaylist'):
+            return self.url_result('5min:%s' % video_id)
 
-            webpage = self._download_webpage(url, playlist_id)
-            title = self._html_search_regex(
-                r'<h1 class="video-title[^"]*">(.+?)</h1>', webpage, 'title')
-            playlist_html = self._search_regex(
-                r"(?s)<ul\s+class='video-related[^']*'>(.*?)</ul>", webpage,
-                'playlist HTML')
-            entries = [{
-                '_type': 'url',
-                'url': 'aol-video:%s' % m.group('id'),
-                'ie_key': 'Aol',
-            } for m in re.finditer(
-                r"<a\s+href='.*videoid=(?P<id>[0-9]+)'\s+class='video-thumb'>",
-                playlist_html)]
+        self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
 
-            return {
-                '_type': 'playlist',
-                'id': playlist_id,
-                'display_id': mobj.group('playlist_display_id'),
-                'title': title,
-                'entries': entries,
-            }
+        webpage = self._download_webpage(url, playlist_id)
+        title = self._html_search_regex(
+            r'<h1 class="video-title[^"]*">(.+?)</h1>', webpage, 'title')
+        playlist_html = self._search_regex(
+            r"(?s)<ul\s+class='video-related[^']*'>(.*?)</ul>", webpage,
+            'playlist HTML')
+        entries = [{
+            '_type': 'url',
+            'url': 'aol-video:%s' % m.group('id'),
+            'ie_key': 'Aol',
+        } for m in re.finditer(
+            r"<a\s+href='.*videoid=(?P<id>[0-9]+)'\s+class='video-thumb'>",
+            playlist_html)]
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'display_id': mobj.group('playlist_display_id'),
+            'title': title,
+            'entries': entries,
+        }
 
-        return FiveMinIE._build_result(video_id)
diff --git a/youtube_dl/extractor/engadget.py b/youtube_dl/extractor/engadget.py
index 92ada81d2..4ea37ebd9 100644
--- a/youtube_dl/extractor/engadget.py
+++ b/youtube_dl/extractor/engadget.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .fivemin import FiveMinIE
 from ..utils import (
     url_basename,
 )
@@ -27,11 +26,10 @@ class EngadgetIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         if video_id is not None:
-            return FiveMinIE._build_result(video_id)
+            return self.url_result('5min:%s' % video_id)
         else:
             title = url_basename(url)
             webpage = self._download_webpage(url, title)
@@ -39,5 +37,5 @@ class EngadgetIE(InfoExtractor):
             return {
                 '_type': 'playlist',
                 'title': title,
-                'entries': [FiveMinIE._build_result(id) for id in ids]
+                'entries': [self.url_result('5min:%s' % vid) for vid in ids]
             }
diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index f9c127ce6..5b24b921c 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -1,11 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
     compat_urllib_parse,
+)
+from ..utils import (
     ExtractorError,
 )
 
@@ -13,7 +13,7 @@ from ..utils import (
 class FiveMinIE(InfoExtractor):
     IE_NAME = '5min'
     _VALID_URL = r'''(?x)
-        (?:https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(.*?&)?playList=|
+        (?:https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(?:.*?&)?playList=|
             5min:)
         (?P<id>\d+)
         '''
@@ -41,13 +41,8 @@ class FiveMinIE(InfoExtractor):
         },
     ]
 
-    @classmethod
-    def _build_result(cls, video_id):
-        return cls.url_result('5min:%s' % video_id, cls.ie_key())
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         embed_url = 'https://embed.5min.com/playerseed/?playList=%s' % video_id
         embed_page = self._download_webpage(embed_url, video_id,
                                             'Downloading embed page')

From 20e35880bf7413c5085c1ade96af0fc6e56d84d1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 12:14:44 +0100
Subject: [PATCH 195/974] [streamcz] Update extractor

---
 youtube_dl/extractor/streamcz.py | 78 +++++++++++++++++---------------
 1 file changed, 41 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 73efe9542..778704839 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -5,14 +5,16 @@ import re
 import json
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+)
 from ..utils import (
     int_or_none,
-    compat_str,
 )
 
 
 class StreamCZIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?stream\.cz/.+/(?P<videoid>.+)'
+    _VALID_URL = r'https?://(?:www\.)?stream\.cz/.+/(?P<id>[0-9]+)'
 
     _TESTS = [{
         'url': 'http://www.stream.cz/peklonataliri/765767-ecka-pro-deti',
@@ -21,61 +23,63 @@ class StreamCZIE(InfoExtractor):
             'id': '765767',
             'ext': 'mp4',
             'title': 'Peklo na talíři: Éčka pro děti',
-            'description': 'md5:49ace0df986e95e331d0fe239d421519',
-            'thumbnail': 'http://im.stream.cz/episode/52961d7e19d423f8f06f0100',
+            'description': 'Taška s grónskou pomazánkou a další pekelnosti ZDE',
+            'thumbnail': 're:^http://im.stream.cz/episode/52961d7e19d423f8f06f0100',
             'duration': 256,
         },
     }, {
         'url': 'http://www.stream.cz/blanik/10002447-tri-roky-pro-mazanka',
-        'md5': '246272e753e26bbace7fcd9deca0650c',
+        'md5': 'e54a254fb8b871968fd8403255f28589',
         'info_dict': {
             'id': '10002447',
             'ext': 'mp4',
             'title': 'Kancelář Blaník: Tři roky pro Mazánka',
-            'description': 'md5:9177695a8b756a0a8ab160de4043b392',
-            'thumbnail': 'http://im.stream.cz/episode/537f838c50c11f8d21320000',
+            'description': 'md5:3862a00ba7bf0b3e44806b544032c859',
+            'thumbnail': 're:^http://im.stream.cz/episode/537f838c50c11f8d21320000',
             'duration': 368,
         },
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-
-        webpage = self._download_webpage(url, video_id)
-
-        data = self._html_search_regex(r'Stream\.Data\.Episode\((.+?)\);', webpage, 'stream data')
-
-        jsonData = json.loads(data)
+        video_id = self._match_id(url)
+        data = self._download_json(
+            'http://www.stream.cz/API/episode/%s' % video_id, video_id)
 
         formats = []
-        for video in jsonData['instances']:
-            for video_format in video['instances']:
-                format_id = video_format['quality']
-
-                if format_id == '240p':
-                    quality = 0
-                elif format_id == '360p':
-                    quality = 1
-                elif format_id == '480p':
-                    quality = 2
-                elif format_id == '720p':
-                    quality = 3
-
+        for quality, video in enumerate(data['video_qualities']):
+            for f in video['formats']:
+                typ = f['type'].partition('/')[2]
+                qlabel = video.get('quality_label')
                 formats.append({
-                    'format_id': '%s-%s' % (video_format['type'].split('/')[1], format_id),
-                    'url': video_format['source'],
+                    'format_note': '%s-%s' % (qlabel, typ) if qlabel else typ,
+                    'format_id': '%s-%s' % (typ, f['quality']),
+                    'url': f['source'],
+                    'height': int_or_none(f['quality'].rstrip('p')),
                     'quality': quality,
                 })
-
         self._sort_formats(formats)
 
+        image = data.get('image')
+        if image:
+            thumbnail = self._proto_relative_url(
+                image.replace('{width}', '1240').replace('{height}', '697'),
+                scheme='http:',
+            )
+        else:
+            thumbnail = None
+
+        stream = data.get('_embedded', {}).get('stream:show', {}).get('name')
+        if stream:
+            title = '%s: %s' % (stream, data['name'])
+        else:
+            title = data['name']
+
         return {
-            'id': compat_str(jsonData['episode_id']),
-            'title': self._og_search_title(webpage),
-            'thumbnail': jsonData['episode_image_original_url'].replace('//', 'http://'),
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
             'formats': formats,
-            'description': self._og_search_description(webpage),
-            'duration': int_or_none(jsonData['duration']),
-            'view_count': int_or_none(jsonData['stats_total']),
+            'description': data.get('web_site_text'),
+            'duration': int_or_none(data.get('duration')),
+            'view_count': int_or_none(data.get('views')),
         }

From 1cc79574fc5df21bf35dccf61eac0e9e75ed8d20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 12:24:42 +0100
Subject: [PATCH 196/974] Fix imports and general cleanup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

· Import from compat what comes from compat. Yes, some names are available in utils too, but that's an implementation detail.
· Use _match_id consistently whenever possible
· Fix some outdated tests
· Use consistent valid URL (always match the whole protocol, no ^ at start required)
· Use modern test definitions
---
 youtube_dl/downloader/common.py              |  2 +-
 youtube_dl/downloader/f4m.py                 |  4 ++-
 youtube_dl/downloader/hls.py                 |  4 ++-
 youtube_dl/downloader/http.py                |  5 +--
 youtube_dl/downloader/rtmp.py                |  2 +-
 youtube_dl/extractor/allocine.py             |  5 +--
 youtube_dl/extractor/aol.py                  |  1 -
 youtube_dl/extractor/appletrailers.py        |  2 +-
 youtube_dl/extractor/auengine.py             |  2 +-
 youtube_dl/extractor/bambuser.py             |  2 +-
 youtube_dl/extractor/bandcamp.py             |  4 ++-
 youtube_dl/extractor/bet.py                  |  2 +-
 youtube_dl/extractor/bilibili.py             |  7 ++--
 youtube_dl/extractor/brightcove.py           | 19 +++++-----
 youtube_dl/extractor/ceskatelevize.py        |  4 ++-
 youtube_dl/extractor/comedycentral.py        |  4 ++-
 youtube_dl/extractor/condenast.py            |  6 ++--
 youtube_dl/extractor/crunchyroll.py          |  6 ++--
 youtube_dl/extractor/dailymotion.py          | 10 +++---
 youtube_dl/extractor/daum.py                 |  2 +-
 youtube_dl/extractor/ehow.py                 | 11 +++---
 youtube_dl/extractor/eighttracks.py          |  2 +-
 youtube_dl/extractor/escapist.py             |  5 +--
 youtube_dl/extractor/everyonesmixtape.py     |  4 ++-
 youtube_dl/extractor/extremetube.py          |  8 +++--
 youtube_dl/extractor/fc2.py                  | 13 ++++---
 youtube_dl/extractor/firedrive.py            | 11 +++---
 youtube_dl/extractor/fourtube.py             | 14 ++++----
 youtube_dl/extractor/franceculture.py        |  2 +-
 youtube_dl/extractor/francetv.py             | 12 ++++---
 youtube_dl/extractor/gamespot.py             |  4 ++-
 youtube_dl/extractor/gdcvault.py             |  2 +-
 youtube_dl/extractor/golem.py                |  4 ++-
 youtube_dl/extractor/googlesearch.py         |  2 +-
 youtube_dl/extractor/gorillavid.py           |  8 ++---
 youtube_dl/extractor/hostingbulk.py          |  8 ++---
 youtube_dl/extractor/hypem.py                | 11 +++---
 youtube_dl/extractor/imdb.py                 |  2 +-
 youtube_dl/extractor/infoq.py                |  7 ++--
 youtube_dl/extractor/internetvideoarchive.py |  4 ++-
 youtube_dl/extractor/iprima.py               |  4 ++-
 youtube_dl/extractor/ivi.py                  |  4 ++-
 youtube_dl/extractor/keezmovies.py           |  7 ++--
 youtube_dl/extractor/livestream.py           |  4 ++-
 youtube_dl/extractor/lynda.py                |  6 ++--
 youtube_dl/extractor/malemotion.py           | 38 +++++++-------------
 youtube_dl/extractor/metacafe.py             |  4 ++-
 youtube_dl/extractor/mit.py                  |  4 ++-
 youtube_dl/extractor/mitele.py               | 13 ++++---
 youtube_dl/extractor/mixcloud.py             |  4 ++-
 youtube_dl/extractor/moevideo.py             |  6 ++--
 youtube_dl/extractor/mofosex.py              |  6 ++--
 youtube_dl/extractor/moniker.py              |  7 ++--
 youtube_dl/extractor/mooshare.py             | 10 +++---
 youtube_dl/extractor/motorsport.py           | 10 +++---
 youtube_dl/extractor/movieclips.py           |  4 ++-
 youtube_dl/extractor/mtv.py                  |  4 ++-
 youtube_dl/extractor/myspass.py              |  5 +--
 youtube_dl/extractor/naver.py                |  8 +++--
 youtube_dl/extractor/nbc.py                  |  4 ++-
 youtube_dl/extractor/nfb.py                  | 14 ++++----
 youtube_dl/extractor/nfl.py                  |  4 ++-
 youtube_dl/extractor/niconico.py             | 10 +++---
 youtube_dl/extractor/noco.py                 | 14 ++++----
 youtube_dl/extractor/nosvideo.py             |  7 ++--
 youtube_dl/extractor/novamov.py              |  4 ++-
 youtube_dl/extractor/nuvid.py                |  9 ++---
 youtube_dl/extractor/photobucket.py          |  5 +--
 youtube_dl/extractor/played.py               |  7 ++--
 youtube_dl/extractor/playfm.py               |  4 ++-
 youtube_dl/extractor/playvid.py              | 24 ++++++-------
 youtube_dl/extractor/pornhub.py              |  8 +++--
 youtube_dl/extractor/promptfile.py           |  8 +++--
 youtube_dl/extractor/prosiebensat1.py        |  4 ++-
 youtube_dl/extractor/quickvid.py             |  4 ++-
 youtube_dl/extractor/rai.py                  |  4 ++-
 youtube_dl/extractor/rts.py                  |  4 ++-
 youtube_dl/extractor/rutube.py               | 13 ++++---
 youtube_dl/extractor/screencast.py           | 11 +++---
 youtube_dl/extractor/shared.py               | 38 +++++++++++---------
 youtube_dl/extractor/sharesix.py             |  4 ++-
 youtube_dl/extractor/sina.py                 |  2 +-
 youtube_dl/extractor/slideshare.py           |  4 ++-
 youtube_dl/extractor/smotri.py               |  4 ++-
 youtube_dl/extractor/sockshare.py            | 15 ++++----
 youtube_dl/extractor/soundcloud.py           |  5 +--
 youtube_dl/extractor/spankwire.py            |  8 +++--
 youtube_dl/extractor/sportdeutschland.py     |  4 ++-
 youtube_dl/extractor/streamcloud.py          |  2 +-
 youtube_dl/extractor/streamcz.py             |  6 ----
 youtube_dl/extractor/tapely.py               |  8 +++--
 youtube_dl/extractor/ted.py                  |  2 +-
 youtube_dl/extractor/theplatform.py          |  4 ++-
 youtube_dl/extractor/tlc.py                  |  2 +-
 youtube_dl/extractor/tube8.py                |  4 ++-
 youtube_dl/extractor/tutv.py                 |  8 ++---
 youtube_dl/extractor/twitch.py               |  4 ++-
 youtube_dl/extractor/udemy.py                |  4 ++-
 youtube_dl/extractor/urort.py                |  6 ++--
 youtube_dl/extractor/ustream.py              |  2 +-
 youtube_dl/extractor/vbox7.py                | 12 +++----
 youtube_dl/extractor/veehd.py                |  9 ++---
 youtube_dl/extractor/veoh.py                 |  4 ++-
 youtube_dl/extractor/vevo.py                 |  4 ++-
 youtube_dl/extractor/videodetective.py       |  9 ++---
 youtube_dl/extractor/videomega.py            | 10 +++---
 youtube_dl/extractor/vk.py                   | 11 +++---
 youtube_dl/extractor/vodlocker.py            |  6 ++--
 youtube_dl/extractor/vube.py                 |  4 ++-
 youtube_dl/extractor/vuclip.py               |  9 ++---
 youtube_dl/extractor/wdr.py                  |  5 +--
 youtube_dl/extractor/wistia.py               |  8 ++---
 youtube_dl/extractor/xbef.py                 |  9 ++---
 youtube_dl/extractor/xnxx.py                 |  9 ++---
 youtube_dl/extractor/xtube.py                |  8 +++--
 youtube_dl/extractor/xvideos.py              | 22 ++++--------
 youtube_dl/extractor/yahoo.py                |  6 ++--
 youtube_dl/extractor/ynet.py                 |  2 +-
 youtube_dl/extractor/youporn.py              |  5 +--
 youtube_dl/swfinterp.py                      |  2 +-
 120 files changed, 443 insertions(+), 384 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 2a566eabe..8181bca09 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -5,8 +5,8 @@ import re
 import sys
 import time
 
+from ..compat import compat_str
 from ..utils import (
-    compat_str,
     encodeFilename,
     format_bytes,
     timeconvert,
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 7cd22c504..00f3a026c 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -9,10 +9,12 @@ import xml.etree.ElementTree as etree
 
 from .common import FileDownloader
 from .http import HttpFD
+from ..compat import (
+    compat_urlparse,
+)
 from ..utils import (
     struct_pack,
     struct_unpack,
-    compat_urlparse,
     format_bytes,
     encodeFilename,
     sanitize_open,
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index ad26cfa40..5bb0f3cfd 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -6,9 +6,11 @@ import subprocess
 
 from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from .common import FileDownloader
-from ..utils import (
+from ..compat import (
     compat_urlparse,
     compat_urllib_request,
+)
+from ..utils import (
     check_executable,
     encodeFilename,
 )
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 224962e86..e68f20c9f 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -4,11 +4,12 @@ import os
 import time
 
 from .common import FileDownloader
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
     compat_urllib_error,
+)
+from ..utils import (
     ContentTooShortError,
-
     encodeFilename,
     sanitize_open,
     format_bytes,
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 58ae2005c..575912675 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -7,9 +7,9 @@ import sys
 import time
 
 from .common import FileDownloader
+from ..compat import compat_str
 from ..utils import (
     check_executable,
-    compat_str,
     encodeFilename,
     format_bytes,
     get_exe_version,
diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 398e93bfb..623aeaf34 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -5,10 +5,9 @@ import re
 import json
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    compat_str,
     qualities,
-    determine_ext,
 )
 
 
@@ -75,9 +74,7 @@ class AllocineIE(InfoExtractor):
                     'format_id': format_id,
                     'quality': quality(format_id),
                     'url': v,
-                    'ext': determine_ext(v),
                 })
-
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index 185ee3693..b51eafc45 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -68,4 +68,3 @@ class AolIE(InfoExtractor):
             'title': title,
             'entries': entries,
         }
-
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 0c01fa1a1..7cd0482c7 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -4,8 +4,8 @@ import re
 import json
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
-    compat_urlparse,
     int_or_none,
 )
 
diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
index 1c765532a..014a21952 100644
--- a/youtube_dl/extractor/auengine.py
+++ b/youtube_dl/extractor/auengine.py
@@ -3,8 +3,8 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
 from ..utils import (
-    compat_urllib_parse,
     determine_ext,
     ExtractorError,
 )
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 1ca0b7cf2..98e1443ab 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -5,7 +5,7 @@ import json
 import itertools
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
 )
 
diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index acddbc8f1..9fb770cb1 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -4,9 +4,11 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
     compat_urlparse,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index c1fc433f7..003e50002 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
 from ..utils import (
-    compat_urllib_parse,
     xpath_text,
     xpath_with_ns,
     int_or_none,
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 0d5889f5d..241b904a9 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -4,8 +4,8 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_parse_qs
 from ..utils import (
-    compat_parse_qs,
     ExtractorError,
     int_or_none,
     unified_strdate,
@@ -29,10 +29,9 @@ class BiliBiliIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         video_code = self._search_regex(
             r'(?s)<div itemprop="video".*?>(.*?)</div>', webpage, 'video code')
 
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index bf18a97e0..1eca00470 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -6,20 +6,21 @@ import json
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
-from ..utils import (
-    compat_urllib_parse,
-    find_xpath_attr,
-    fix_xml_ampersands,
-    compat_urlparse,
-    compat_str,
-    compat_urllib_request,
+from ..compat import (
     compat_parse_qs,
+    compat_str,
+    compat_urllib_parse,
     compat_urllib_parse_urlparse,
-
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
     determine_ext,
     ExtractorError,
-    unsmuggle_url,
+    find_xpath_attr,
+    fix_xml_ampersands,
     unescapeHTML,
+    unsmuggle_url,
 )
 
 
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 97feb6704..2f866f3ef 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -4,10 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 2e3ef3fda..48e2410b6 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .mtv import MTVServicesInfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
     compat_urllib_parse,
+)
+from ..utils import (
     ExtractorError,
     float_or_none,
     unified_strdate,
diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 7a7e79360..3db4db4e4 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -5,12 +5,14 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
-    orderedSet,
     compat_urllib_parse_urlparse,
     compat_urlparse,
 )
+from ..utils import (
+    orderedSet,
+)
 
 
 class CondeNastIE(InfoExtractor):
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index d7e2b841e..8f1ea02e7 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -10,10 +10,12 @@ import xml.etree.ElementTree
 from hashlib import sha1
 from math import pow, sqrt, floor
 from .subtitles import SubtitlesInfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
     bytes_to_intlist,
     intlist_to_bytes,
     unified_strdate,
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 936c13cd6..cf5841a7c 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -8,13 +8,15 @@ import itertools
 from .common import InfoExtractor
 from .subtitles import SubtitlesInfoExtractor
 
-from ..utils import (
-    compat_urllib_request,
+from ..compat import (
     compat_str,
+    compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
     orderedSet,
     str_to_int,
-    int_or_none,
-    ExtractorError,
     unescapeHTML,
 )
 
diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 45d66e2e6..c6b813f58 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
diff --git a/youtube_dl/extractor/ehow.py b/youtube_dl/extractor/ehow.py
index b766e17f2..9cb1bf301 100644
--- a/youtube_dl/extractor/ehow.py
+++ b/youtube_dl/extractor/ehow.py
@@ -1,8 +1,6 @@
 from __future__ import unicode_literals
 
-import re
-
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 from .common import InfoExtractor
@@ -24,11 +22,10 @@ class EHowIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        video_url = self._search_regex(r'(?:file|source)=(http[^\'"&]*)',
-                                       webpage, 'video URL')
+        video_url = self._search_regex(
+            r'(?:file|source)=(http[^\'"&]*)', webpage, 'video URL')
         final_url = compat_urllib_parse.unquote(video_url)
         uploader = self._html_search_meta('uploader', webpage)
         title = self._og_search_title(webpage).replace(' | eHow', '')
diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index f4c1e2a72..a30a1f330 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -6,7 +6,7 @@ import random
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
 )
 
diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 476fc22b9..e240cb859 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -3,9 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/everyonesmixtape.py b/youtube_dl/extractor/everyonesmixtape.py
index d237a8281..d872d828f 100644
--- a/youtube_dl/extractor/everyonesmixtape.py
+++ b/youtube_dl/extractor/everyonesmixtape.py
@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index aacbf1414..36ba33128 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -3,16 +3,18 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urllib_parse,
+)
+from ..utils import (
     str_to_int,
 )
 
 
 class ExtremeTubeIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>extremetube\.com/.*?video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>extremetube\.com/.*?video/.+?(?P<id>[0-9]+))(?:[/?&]|$)'
     _TESTS = [{
         'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
         'md5': '1fb9228f5e3332ec8c057d6ac36f33e0',
@@ -31,7 +33,7 @@ class ExtremeTubeIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
+        video_id = mobj.group('id')
         url = 'http://www.' + mobj.group('url')
 
         req = compat_urllib_request.Request(url)
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 6f5d23559..81ceace53 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -1,19 +1,20 @@
 #! -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
 import hashlib
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_request,
     compat_urlparse,
 )
+from ..utils import (
+    ExtractorError,
+)
 
 
 class FC2IE(InfoExtractor):
-    _VALID_URL = r'^http://video\.fc2\.com/((?P<lang>[^/]+)/)?content/(?P<id>[^/]+)'
+    _VALID_URL = r'^http://video\.fc2\.com/(?:[^/]+/)?content/(?P<id>[^/]+)'
     IE_NAME = 'fc2'
     _TEST = {
         'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
@@ -26,9 +27,7 @@ class FC2IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         self._downloader.cookiejar.clear_session_cookies()  # must clear
 
diff --git a/youtube_dl/extractor/firedrive.py b/youtube_dl/extractor/firedrive.py
index af439ccfe..3191116d9 100644
--- a/youtube_dl/extractor/firedrive.py
+++ b/youtube_dl/extractor/firedrive.py
@@ -4,11 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import (
+    ExtractorError,
+)
 
 
 class FiredriveIE(InfoExtractor):
@@ -28,11 +30,8 @@ class FiredriveIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         url = 'http://firedrive.com/file/%s' % video_id
-
         webpage = self._download_webpage(url, video_id)
 
         if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index b22ce2acb..7187e0752 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -3,12 +3,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
-    unified_strdate,
-    str_to_int,
-    parse_duration,
+)
+from ..utils import (
     clean_html,
+    parse_duration,
+    str_to_int,
+    unified_strdate,
 )
 
 
@@ -31,9 +33,7 @@ class FourTubeIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage_url = 'http://www.4tube.com/videos/' + video_id
         webpage = self._download_webpage(webpage_url, video_id)
 
diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index 898e0dda7..0c2972162 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -5,7 +5,7 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_parse_qs,
     compat_urlparse,
 )
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index e0420a48f..bbc760a49 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -6,13 +6,15 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
-    compat_urlparse,
-    ExtractorError,
-    clean_html,
-    parse_duration,
+from ..compat import (
     compat_urllib_parse_urlparse,
+    compat_urlparse,
+)
+from ..utils import (
+    clean_html,
+    ExtractorError,
     int_or_none,
+    parse_duration,
 )
 
 
diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index d570e3f6a..47373e215 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -4,9 +4,11 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
+)
+from ..utils import (
     unescapeHTML,
 )
 
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index de14ae1fb..d453ec010 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
diff --git a/youtube_dl/extractor/golem.py b/youtube_dl/extractor/golem.py
index 53714f47f..2bfb99040 100644
--- a/youtube_dl/extractor/golem.py
+++ b/youtube_dl/extractor/golem.py
@@ -2,8 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
+)
+from ..utils import (
     determine_ext,
 )
 
diff --git a/youtube_dl/extractor/googlesearch.py b/youtube_dl/extractor/googlesearch.py
index 469e1f935..498304cb2 100644
--- a/youtube_dl/extractor/googlesearch.py
+++ b/youtube_dl/extractor/googlesearch.py
@@ -4,7 +4,7 @@ import itertools
 import re
 
 from .common import SearchInfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 1ac1da856..ae24aff84 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -4,11 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    determine_ext,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
     int_or_none,
 )
 
@@ -106,7 +107,6 @@ class GorillaVidIE(InfoExtractor):
         formats = [{
             'format_id': 'sd',
             'url': video_url,
-            'ext': determine_ext(video_url),
             'quality': 1,
         }]
 
diff --git a/youtube_dl/extractor/hostingbulk.py b/youtube_dl/extractor/hostingbulk.py
index 8e812b669..704d0285d 100644
--- a/youtube_dl/extractor/hostingbulk.py
+++ b/youtube_dl/extractor/hostingbulk.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+)
 from ..utils import (
     ExtractorError,
-    compat_urllib_request,
     int_or_none,
     urlencode_postdata,
 )
@@ -30,9 +32,7 @@ class HostingBulkIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         url = 'http://hostingbulk.com/{0:}.html'.format(video_id)
 
         # Custom request with cookie to set language to English, so our file
diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index 6d0d847c6..aa0724a02 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -1,20 +1,20 @@
 from __future__ import unicode_literals
 
 import json
-import re
 import time
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
 
 class HypemIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?hypem\.com/track/([^/]+)/([^/]+)'
+    _VALID_URL = r'http://(?:www\.)?hypem\.com/track/(?P<id>[^/]+)/'
     _TEST = {
         'url': 'http://hypem.com/track/1v6ga/BODYWORK+-+TAME',
         'md5': 'b9cc91b5af8995e9f0c1cee04c575828',
@@ -27,8 +27,7 @@ class HypemIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        track_id = mobj.group(1)
+        track_id = self._match_id(url)
 
         data = {'ax': 1, 'ts': time.time()}
         data_encoded = compat_urllib_parse.urlencode(data)
diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index f2c1c10f5..13a53a0cb 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -4,7 +4,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
 )
 
diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index e76dd222d..f25f43664 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
 import base64
-import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
@@ -24,9 +23,7 @@ class InfoQIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py
index 1e4799187..c813d4b82 100644
--- a/youtube_dl/extractor/internetvideoarchive.py
+++ b/youtube_dl/extractor/internetvideoarchive.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
     compat_urllib_parse,
+)
+from ..utils import (
     xpath_with_ns,
 )
 
diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 4247d6391..8529bedfc 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -6,8 +6,10 @@ from random import random
 from math import floor
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index f0fba1adb..7a400323d 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -5,8 +5,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index 75b63cffb..97dcb518a 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -4,7 +4,7 @@ import os
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urllib_parse,
@@ -15,7 +15,7 @@ from ..aes import (
 
 
 class KeezMoviesIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?keezmovies\.com/video/.+?(?P<videoid>[0-9]+)(?:[/?&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/.+?(?P<id>[0-9]+)(?:[/?&]|$)'
     _TEST = {
         'url': 'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
         'file': '1214711.mp4',
@@ -27,8 +27,7 @@ class KeezMoviesIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
+        video_id = self._match_id(url)
 
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'age_verified=1')
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 03c4691c6..5247c6f58 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -4,10 +4,12 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
     compat_urllib_parse_urlparse,
     compat_urlparse,
+)
+from ..utils import (
     ExtractorError,
     find_xpath_attr,
     int_or_none,
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 2160d6cb0..26e84970d 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -5,12 +5,14 @@ import json
 
 from .subtitles import SubtitlesInfoExtractor
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
+    compat_str,
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     int_or_none,
-    compat_str,
 )
 
 
diff --git a/youtube_dl/extractor/malemotion.py b/youtube_dl/extractor/malemotion.py
index 1abf6e4f8..0b85a59d1 100644
--- a/youtube_dl/extractor/malemotion.py
+++ b/youtube_dl/extractor/malemotion.py
@@ -1,43 +1,33 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
 
 class MalemotionIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?malemotion\.com/video/(.+?)\.(?P<id>.+?)(#|$)'
+    _VALID_URL = r'https?://malemotion\.com/video/(.+?)\.(?P<id>.+?)(#|$)'
     _TEST = {
-        'url': 'http://malemotion.com/video/bien-dur.10ew',
-        'file': '10ew.mp4',
-        'md5': 'b3cc49f953b107e4a363cdff07d100ce',
+        'url': 'http://malemotion.com/video/bete-de-concours.ltc',
+        'md5': '3013e53a0afbde2878bc39998c33e8a5',
         'info_dict': {
-            "title": "Bien dur",
-            "age_limit": 18,
+            'id': 'ltc',
+            'ext': 'mp4',
+            'title': 'Bête de Concours',
+            'age_limit': 18,
         },
-        'skip': 'This video has been deleted.'
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group("id")
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
-
-        # Extract video URL
-        video_url = compat_urllib_parse.unquote(
-            self._search_regex(r'<source type="video/mp4" src="(.+?)"', webpage, 'video URL'))
-
-        # Extract title
+        video_url = compat_urllib_parse.unquote(self._search_regex(
+            r'<source type="video/mp4" src="(.+?)"', webpage, 'video URL'))
         video_title = self._html_search_regex(
             r'<title>(.*?)</title', webpage, 'title')
-
-        # Extract video thumbnail
         video_thumbnail = self._search_regex(
             r'<video .+?poster="(.+?)"', webpage, 'thumbnail', fatal=False)
 
@@ -47,14 +37,12 @@ class MalemotionIE(InfoExtractor):
             'format_id': 'mp4',
             'preference': 1,
         }]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'formats': formats,
-            'uploader': None,
-            'upload_date': None,
             'title': video_title,
             'thumbnail': video_thumbnail,
-            'description': None,
             'age_limit': 18,
         }
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 858c1c0c3..8bc333b02 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -3,10 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_parse_qs,
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 807b1dc89..d354702d1 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -5,8 +5,10 @@ import json
 
 from .common import InfoExtractor
 from .youtube import YoutubeIE
-from ..utils import (
+from ..compat import (
     compat_urlparse,
+)
+from ..utils import (
     clean_html,
     ExtractorError,
     get_element_by_id,
diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 6691521e5..256758323 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -1,12 +1,13 @@
 from __future__ import unicode_literals
 
-import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
+)
+from ..utils import (
     get_element_by_attribute,
     parse_duration,
     strip_jsonp,
@@ -15,7 +16,7 @@ from ..utils import (
 
 class MiTeleIE(InfoExtractor):
     IE_NAME = 'mitele.es'
-    _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<episode>[^/]+)/'
+    _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
 
     _TEST = {
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
@@ -31,12 +32,10 @@ class MiTeleIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        episode = mobj.group('episode')
+        episode = self._match_id(url)
         webpage = self._download_webpage(url, episode)
         embed_data_json = self._search_regex(
-            r'MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data',
-            flags=re.DOTALL
+            r'(?s)MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data',
         ).replace('\'', '"')
         embed_data = json.loads(embed_data_json)
 
diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 55cc33a3e..07d194562 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
+)
+from ..utils import (
     ExtractorError,
     HEADRequest,
     int_or_none,
diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dl/extractor/moevideo.py
index 2ff79b9b8..184f9c2c9 100644
--- a/youtube_dl/extractor/moevideo.py
+++ b/youtube_dl/extractor/moevideo.py
@@ -5,10 +5,12 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
     int_or_none,
 )
 
diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index d658647e6..2cec12d35 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -4,7 +4,7 @@ import os
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urllib_parse,
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class MofosexIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?(?P<url>mofosex\.com/videos/(?P<videoid>[0-9]+)/.*?\.html)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>mofosex\.com/videos/(?P<id>[0-9]+)/.*?\.html)'
     _TEST = {
         'url': 'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html',
         'md5': '1b2eb47ac33cc75d4a80e3026b613c5a',
@@ -26,7 +26,7 @@ class MofosexIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
+        video_id = mobj.group('id')
         url = 'http://www.' + mobj.group('url')
 
         req = compat_urllib_request.Request(url)
diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 1c4f589cc..5de719bdc 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -5,7 +5,7 @@ import os.path
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
@@ -37,10 +37,9 @@ class MonikerIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         orig_webpage = self._download_webpage(url, video_id)
+
         fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
         data = dict(fields)
 
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 34a4bec3a..9f2853fa3 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -4,11 +4,13 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
 )
+from ..utils import (
+    ExtractorError,
+)
 
 
 class MooshareIE(InfoExtractor):
@@ -43,9 +45,7 @@ class MooshareIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         page = self._download_webpage(url, video_id, 'Downloading page')
 
         if re.search(r'>Video Not Found or Deleted<', page) is not None:
diff --git a/youtube_dl/extractor/motorsport.py b/youtube_dl/extractor/motorsport.py
index 7c0ec6a12..f5ca74e97 100644
--- a/youtube_dl/extractor/motorsport.py
+++ b/youtube_dl/extractor/motorsport.py
@@ -3,13 +3,14 @@ from __future__ import unicode_literals
 
 import hashlib
 import json
-import re
 import time
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_parse_qs,
     compat_str,
+)
+from ..utils import (
     int_or_none,
 )
 
@@ -32,10 +33,9 @@ class MotorsportIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-
+        display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
+
         flashvars_code = self._html_search_regex(
             r'<embed id="player".*?flashvars="([^"]+)"', webpage, 'flashvars')
         flashvars = compat_parse_qs(flashvars_code)
diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index 456807dd1..04e17d055 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+)
 from ..utils import (
     ExtractorError,
-    compat_str,
     clean_html,
 )
 
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index b482d6d4d..5ebc78033 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index 51e540814..5b9b9fbcd 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -2,9 +2,10 @@ from __future__ import unicode_literals
 import os.path
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index fbe34defd..c10405f04 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -4,8 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
+)
+from ..utils import (
     ExtractorError,
     clean_html,
 )
@@ -26,9 +28,9 @@ class NaverIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
                          webpage)
         if m_id is None:
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index bf5132721..690c46b6a 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -4,8 +4,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
+)
+from ..utils import (
     ExtractorError,
     find_xpath_attr,
 )
diff --git a/youtube_dl/extractor/nfb.py b/youtube_dl/extractor/nfb.py
index 7ce1d481d..ea077254b 100644
--- a/youtube_dl/extractor/nfb.py
+++ b/youtube_dl/extractor/nfb.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
 )
@@ -12,7 +10,7 @@ from ..utils import (
 class NFBIE(InfoExtractor):
     IE_NAME = 'nfb'
     IE_DESC = 'National Film Board of Canada'
-    _VALID_URL = r'https?://(?:www\.)?(nfb|onf)\.ca/film/(?P<id>[\da-z_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:nfb|onf)\.ca/film/(?P<id>[\da-z_-]+)'
 
     _TEST = {
         'url': 'https://www.nfb.ca/film/qallunaat_why_white_people_are_funny',
@@ -32,10 +30,10 @@ class NFBIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        page = self._download_webpage('https://www.nfb.ca/film/%s' % video_id, video_id, 'Downloading film page')
+        video_id = self._match_id(url)
+        page = self._download_webpage(
+            'https://www.nfb.ca/film/%s' % video_id, video_id,
+            'Downloading film page')
 
         uploader_id = self._html_search_regex(r'<a class="director-link" href="/explore-all-directors/([^/]+)/"',
                                               page, 'director id', fatal=False)
diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index cc7c921c3..606e2294e 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
     ExtractorError,
-    compat_urllib_parse_urlparse,
     int_or_none,
     remove_end,
 )
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 1d9c1a096..4c1890416 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -5,14 +5,16 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
-    unified_strdate,
-    parse_duration,
-    int_or_none,
+)
+from ..utils import (
     ExtractorError,
+    int_or_none,
+    parse_duration,
+    unified_strdate,
 )
 
 
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 7d2ff7b9a..251e6da07 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -6,13 +6,15 @@ import time
 import hashlib
 
 from .common import InfoExtractor
-from ..utils import (
-    compat_urllib_request,
-    compat_urllib_parse,
-    ExtractorError,
-    clean_html,
-    unified_strdate,
+from ..compat import (
     compat_str,
+    compat_urllib_parse,
+    compat_urllib_request,
+)
+from ..utils import (
+    clean_html,
+    ExtractorError,
+    unified_strdate,
 )
 
 
diff --git a/youtube_dl/extractor/nosvideo.py b/youtube_dl/extractor/nosvideo.py
index f3be8f552..f5ef856db 100644
--- a/youtube_dl/extractor/nosvideo.py
+++ b/youtube_dl/extractor/nosvideo.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+)
 from ..utils import (
     ExtractorError,
-    compat_urllib_request,
     urlencode_postdata,
     xpath_text,
     xpath_with_ns,
@@ -32,8 +34,7 @@ class NosVideoIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         fields = {
             'id': video_id,
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 38d05e466..04d779890 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
-    compat_urlparse
 )
 
 
diff --git a/youtube_dl/extractor/nuvid.py b/youtube_dl/extractor/nuvid.py
index 449c8a6a3..57928f2ae 100644
--- a/youtube_dl/extractor/nuvid.py
+++ b/youtube_dl/extractor/nuvid.py
@@ -3,15 +3,17 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+)
 from ..utils import (
     parse_duration,
     unified_strdate,
-    compat_urllib_request,
 )
 
 
 class NuvidIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www|m)\.nuvid\.com/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www|m)\.nuvid\.com/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://m.nuvid.com/video/1310741/',
         'md5': 'eab207b7ac4fccfb4e23c86201f11277',
@@ -26,8 +28,7 @@ class NuvidIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         formats = []
 
diff --git a/youtube_dl/extractor/photobucket.py b/youtube_dl/extractor/photobucket.py
index b4389e0b6..c66db3cdc 100644
--- a/youtube_dl/extractor/photobucket.py
+++ b/youtube_dl/extractor/photobucket.py
@@ -4,16 +4,17 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..utils import compat_urllib_parse
+from ..compat import compat_urllib_parse
 
 
 class PhotobucketIE(InfoExtractor):
     _VALID_URL = r'http://(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))'
     _TEST = {
         'url': 'http://media.photobucket.com/user/rachaneronas/media/TiredofLinkBuildingTryBacklinkMyDomaincom_zpsc0c3b9fa.mp4.html?filters[term]=search&filters[primary]=videos&filters[secondary]=images&sort=1&o=0',
-        'file': 'zpsc0c3b9fa.mp4',
         'md5': '7dabfb92b0a31f6c16cebc0f8e60ff99',
         'info_dict': {
+            'id': 'zpsc0c3b9fa',
+            'ext': 'mp4',
             'timestamp': 1367669341,
             'upload_date': '20130504',
             'uploader': 'rachaneronas',
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 17880471d..449d4836c 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -5,11 +5,13 @@ import re
 import os.path
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import (
+    ExtractorError,
+)
 
 
 class PlayedIE(InfoExtractor):
@@ -28,7 +30,6 @@ class PlayedIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         orig_webpage = self._download_webpage(url, video_id)
 
         m_error = re.search(
diff --git a/youtube_dl/extractor/playfm.py b/youtube_dl/extractor/playfm.py
index ebc046804..9576aed0e 100644
--- a/youtube_dl/extractor/playfm.py
+++ b/youtube_dl/extractor/playfm.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
diff --git a/youtube_dl/extractor/playvid.py b/youtube_dl/extractor/playvid.py
index cd3905acb..c3e667e9e 100644
--- a/youtube_dl/extractor/playvid.py
+++ b/youtube_dl/extractor/playvid.py
@@ -3,31 +3,31 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    clean_html,
+from ..compat import (
     compat_urllib_parse,
 )
+from ..utils import (
+    clean_html,
+    ExtractorError,
+)
 
 
 class PlayvidIE(InfoExtractor):
-    _VALID_URL = r'^https?://www\.playvid\.com/watch(\?v=|/)(?P<id>.+?)(?:#|$)'
+    _VALID_URL = r'https?://www\.playvid\.com/watch(\?v=|/)(?P<id>.+?)(?:#|$)'
     _TEST = {
-        'url': 'http://www.playvid.com/watch/agbDDi7WZTV',
-        'md5': '44930f8afa616efdf9482daf4fe53e1e',
+        'url': 'http://www.playvid.com/watch/RnmBNgtrrJu',
+        'md5': 'ffa2f6b2119af359f544388d8c01eb6c',
         'info_dict': {
-            'id': 'agbDDi7WZTV',
+            'id': 'RnmBNgtrrJu',
             'ext': 'mp4',
-            'title': 'Michelle Lewin in Miami Beach',
-            'duration': 240,
+            'title': 'md5:9256d01c6317e3f703848b5906880dc8',
+            'duration': 82,
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         m_error = re.search(
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 2ca15b717..634142d0d 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -4,10 +4,12 @@ import os
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
+    compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
+)
+from ..utils import (
     str_to_int,
 )
 from ..aes import (
@@ -16,7 +18,7 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?pornhub\.com/view_video\.php\?viewkey=(?P<id>[0-9a-f]+)'
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/view_video\.php\?viewkey=(?P<id>[0-9a-f]+)'
     _TEST = {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': '882f488fa1f0026f023f33576004a2ed',
diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index 7fcde086c..f536e6e6c 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -4,12 +4,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    determine_ext,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import (
+    determine_ext,
+    ExtractorError,
+)
 
 
 class PromptFileIE(InfoExtractor):
diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 1262793c8..385681d06 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -5,8 +5,10 @@ import re
 
 from hashlib import sha1
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
+)
+from ..utils import (
     unified_strdate,
 )
 
diff --git a/youtube_dl/extractor/quickvid.py b/youtube_dl/extractor/quickvid.py
index 3bc78060d..af7d76cf4 100644
--- a/youtube_dl/extractor/quickvid.py
+++ b/youtube_dl/extractor/quickvid.py
@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
+)
+from ..utils import (
     determine_ext,
     int_or_none,
 )
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 2d39ecfe4..aa26b7e0b 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -3,10 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .subtitles import SubtitlesInfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+)
 from ..utils import (
     parse_duration,
     unified_strdate,
-    compat_urllib_parse,
 )
 
 
diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index dc59a5e5c..5e84c1098 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -4,12 +4,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+)
 from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
     unescapeHTML,
-    compat_str,
 )
 
 
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 6941d96fb..b72b5a586 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -5,10 +5,12 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
-    unified_strdate,
+)
+from ..utils import (
     ExtractorError,
+    unified_strdate,
 )
 
 
@@ -36,9 +38,7 @@ class RutubeIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         video = self._download_json(
             'http://rutube.ru/api/video/%s/?format=json' % video_id,
             video_id, 'Downloading video JSON')
@@ -114,8 +114,7 @@ class RutubeMovieIE(RutubeChannelIE):
     _PAGE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/video?page=%s&format=json'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        movie_id = mobj.group('id')
+        movie_id = self._match_id(url)
         movie = self._download_json(
             self._MOVIE_TEMPLATE % movie_id, movie_id,
             'Downloading movie JSON')
diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py
index c145f6fc7..dfd897ba3 100644
--- a/youtube_dl/extractor/screencast.py
+++ b/youtube_dl/extractor/screencast.py
@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_parse_qs,
     compat_urllib_request,
 )
+from ..utils import (
+    ExtractorError,
+)
 
 
 class ScreencastIE(InfoExtractor):
@@ -57,8 +57,7 @@ class ScreencastIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._html_search_regex(
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index fdc31603a..26ced716e 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -4,10 +4,12 @@ import re
 import base64
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
+)
 from ..utils import (
     ExtractorError,
-    compat_urllib_request,
-    compat_urllib_parse,
     int_or_none,
 )
 
@@ -26,26 +28,30 @@ class SharedIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
 
-        page = self._download_webpage(url, video_id)
+        if '>File does not exist<' in webpage:
+            raise ExtractorError(
+                'Video %s does not exist' % video_id, expected=True)
 
-        if re.search(r'>File does not exist<', page) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
-        download_form = dict(re.findall(r'<input type="hidden" name="([^"]+)" value="([^"]*)"', page))
-
-        request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(download_form))
+        download_form = dict(re.findall(
+            r'<input type="hidden" name="([^"]+)" value="([^"]*)"', webpage))
+        request = compat_urllib_request.Request(
+            url, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
-        video_page = self._download_webpage(request, video_id, 'Downloading video page')
+        video_page = self._download_webpage(
+            request, video_id, 'Downloading video page')
 
-        video_url = self._html_search_regex(r'data-url="([^"]+)"', video_page, 'video URL')
-        title = base64.b64decode(self._html_search_meta('full:title', page, 'title')).decode('utf-8')
-        filesize = int_or_none(self._html_search_meta('full:size', page, 'file size', fatal=False))
+        video_url = self._html_search_regex(
+            r'data-url="([^"]+)"', video_page, 'video URL')
+        title = base64.b64decode(self._html_search_meta(
+            'full:title', webpage, 'title')).decode('utf-8')
+        filesize = int_or_none(self._html_search_meta(
+            'full:size', webpage, 'file size', fatal=False))
         thumbnail = self._html_search_regex(
-            r'data-poster="([^"]+)"', video_page, 'thumbnail', fatal=False, default=None)
+            r'data-poster="([^"]+)"', video_page, 'thumbnail', default=None)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/sharesix.py b/youtube_dl/extractor/sharesix.py
index 7531e8325..ac3e3adf2 100644
--- a/youtube_dl/extractor/sharesix.py
+++ b/youtube_dl/extractor/sharesix.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     parse_duration,
 )
 
diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py
index 5eadbb7ea..a63d126d4 100644
--- a/youtube_dl/extractor/sina.py
+++ b/youtube_dl/extractor/sina.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
 )
diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index 5864b9936..e7d776e7b 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -4,8 +4,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 646af3cc9..d031fe401 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -7,9 +7,11 @@ import hashlib
 import uuid
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     int_or_none,
     unified_strdate,
diff --git a/youtube_dl/extractor/sockshare.py b/youtube_dl/extractor/sockshare.py
index c663e56d4..7d3c0e937 100644
--- a/youtube_dl/extractor/sockshare.py
+++ b/youtube_dl/extractor/sockshare.py
@@ -1,13 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from ..utils import (
-    ExtractorError,
+import re
+
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
-    determine_ext,
 )
-import re
+from ..utils import (
+    determine_ext,
+    ExtractorError,
+)
 
 from .common import InfoExtractor
 
@@ -27,9 +30,7 @@ class SockshareIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         url = 'http://sockshare.com/file/%s' % video_id
         webpage = self._download_webpage(url, video_id)
 
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index ab9483d2d..5d60c4939 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -5,11 +5,12 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
     compat_urlparse,
     compat_urllib_parse,
-
+)
+from ..utils import (
     ExtractorError,
     int_or_none,
     unified_strdate,
diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 94602e89e..b936202f6 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -3,12 +3,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
+    compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
-    unified_strdate,
+)
+from ..utils import (
     str_to_int,
+    unified_strdate,
 )
 from ..aes import aes_decrypt_text
 
diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index 057ef5251..2f57f5b7c 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -4,8 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     parse_iso8601,
 )
 
diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index c1178f26d..38176498d 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -5,7 +5,7 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 778704839..c3ceb5f76 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -1,13 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-)
 from ..utils import (
     int_or_none,
 )
diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
index 283e11350..f1f43d0a7 100644
--- a/youtube_dl/extractor/tapely.py
+++ b/youtube_dl/extractor/tapely.py
@@ -4,10 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    clean_html,
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
+    clean_html,
+    ExtractorError,
     float_or_none,
     parse_iso8601,
 )
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 72160503c..944177426 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -5,7 +5,7 @@ import re
 
 from .subtitles import SubtitlesInfoExtractor
 
-from ..utils import (
+from ..compat import (
     compat_str,
 )
 
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index e2653d62d..af6ef0033 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -4,8 +4,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
+)
+from ..utils import (
     determine_ext,
     ExtractorError,
     xpath_with_ns,
diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index 66d159e99..9f9e388c5 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -5,7 +5,7 @@ import re
 from .common import InfoExtractor
 from .brightcove import BrightcoveIE
 from .discovery import DiscoveryIE
-from ..utils import compat_urlparse
+from ..compat import compat_urlparse
 
 
 class TlcIE(DiscoveryIE):
diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 64a1e9030..d73ad3762 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -4,9 +4,11 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
+)
+from ..utils import (
     int_or_none,
     str_to_int,
 )
diff --git a/youtube_dl/extractor/tutv.py b/youtube_dl/extractor/tutv.py
index d516b6427..4de0aac52 100644
--- a/youtube_dl/extractor/tutv.py
+++ b/youtube_dl/extractor/tutv.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
 import base64
-import re
 
 from .common import InfoExtractor
-from ..utils import compat_parse_qs
+from ..compat import compat_parse_qs
 
 
 class TutvIE(InfoExtractor):
@@ -20,10 +19,9 @@ class TutvIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         internal_id = self._search_regex(r'codVideo=([0-9]+)', webpage, 'internal video ID')
 
         data_content = self._download_webpage(
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 397d167e8..715f9930c 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -5,9 +5,11 @@ import itertools
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     parse_iso8601,
 )
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 5271611ac..4667ed83b 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/urort.py b/youtube_dl/extractor/urort.py
index 249a34c86..8872cfcb2 100644
--- a/youtube_dl/extractor/urort.py
+++ b/youtube_dl/extractor/urort.py
@@ -1,11 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
+)
+from ..utils import (
     unified_strdate,
 )
 
diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 53dc3a496..68d03b999 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
 )
 
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 455b6d9da..dd026748d 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -1,19 +1,18 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
 
 class Vbox7IE(InfoExtractor):
-    _VALID_URL = r'http://(www\.)?vbox7\.com/play:(?P<id>[^/]+)'
+    _VALID_URL = r'http://(?:www\.)?vbox7\.com/play:(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://vbox7.com/play:249bb972c2',
         'md5': '99f65c0c9ef9b682b97313e052734c3f',
@@ -25,8 +24,7 @@ class Vbox7IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         redirect_page, urlh = self._download_webpage_handle(url, video_id)
         new_location = self._search_regex(r'window\.location = \'(.*)\';',
diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 94647d1c8..815f58468 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -4,10 +4,12 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
-    get_element_by_id,
+)
+from ..utils import (
     clean_html,
+    get_element_by_id,
 )
 
 
@@ -26,8 +28,7 @@ class VeeHDIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         # VeeHD seems to send garbage on the first request.
         # See https://github.com/rg3/youtube-dl/issues/2102
diff --git a/youtube_dl/extractor/veoh.py b/youtube_dl/extractor/veoh.py
index a7953a7e7..01e258e32 100644
--- a/youtube_dl/extractor/veoh.py
+++ b/youtube_dl/extractor/veoh.py
@@ -4,8 +4,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     int_or_none,
     ExtractorError,
 )
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index c912c3cbe..43f6b029d 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -4,8 +4,10 @@ import re
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/videodetective.py b/youtube_dl/extractor/videodetective.py
index ac6c25537..0ffc7ff7d 100644
--- a/youtube_dl/extractor/videodetective.py
+++ b/youtube_dl/extractor/videodetective.py
@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from .internetvideoarchive import InternetVideoArchiveIE
-from ..utils import compat_urlparse
 
 
 class VideoDetectiveIE(InfoExtractor):
@@ -17,13 +15,12 @@ class VideoDetectiveIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'KICK-ASS 2',
             'description': 'md5:65ba37ad619165afac7d432eaded6013',
-            'duration': 135,
+            'duration': 138,
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         og_video = self._og_search_video_url(webpage)
         query = compat_urlparse.urlparse(og_video).query
diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 29c4e0101..7a78f0d26 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -1,11 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
+)
+from ..utils import (
     remove_start,
 )
 
@@ -27,9 +27,7 @@ class VideoMegaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         url = 'http://videomega.tv/iframe.php?ref={0:}'.format(video_id)
         webpage = self._download_webpage(url, video_id)
 
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index ca6b0d5b3..542e9198a 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -5,14 +5,17 @@ import re
 import json
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+    compat_urllib_parse,
+    compat_urllib_request,
+)
 from ..utils import (
     ExtractorError,
-    compat_urllib_request,
-    compat_urllib_parse,
-    compat_str,
+    orderedSet,
     unescapeHTML,
     unified_strdate,
-    orderedSet)
+)
 
 
 class VKIE(InfoExtractor):
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index affef6507..1c0966a79 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -2,8 +2,9 @@
 from __future__ import unicode_literals
 
 import re
+
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
@@ -24,8 +25,7 @@ class VodlockerIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         fields = dict(re.findall(r'''(?x)<input\s+
diff --git a/youtube_dl/extractor/vube.py b/youtube_dl/extractor/vube.py
index 1b2f731e9..405cb9db4 100644
--- a/youtube_dl/extractor/vube.py
+++ b/youtube_dl/extractor/vube.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+)
 from ..utils import (
     int_or_none,
-    compat_str,
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index ec3c010ad..c3fde53f5 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
+)
+from ..utils import (
     ExtractorError,
     parse_duration,
     qualities,
@@ -25,10 +27,9 @@ class VuClipIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         ad_m = re.search(
             r'''value="No.*?" onClick="location.href='([^"']+)'"''', webpage)
         if ad_m:
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 93a6e6454..8e25ecf28 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_parse_qs,
     compat_urlparse,
+)
+from ..utils import (
     determine_ext,
     unified_strdate,
 )
@@ -141,7 +143,6 @@ class WDRMobileIE(InfoExtractor):
             'title': mobj.group('title'),
             'age_limit': int(mobj.group('age_limit')),
             'url': url,
-            'ext': determine_ext(url),
             'user_agent': 'mobile',
         }
 
diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 748443f81..13a079151 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -1,9 +1,8 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import ExtractorError, compat_urllib_request
+from ..compat import compat_urllib_request
+from ..utils import ExtractorError
 
 
 class WistiaIE(InfoExtractor):
@@ -22,8 +21,7 @@ class WistiaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         request = compat_urllib_request.Request(self._API_URL.format(video_id))
         request.add_header('Referer', url)  # Some videos require this.
diff --git a/youtube_dl/extractor/xbef.py b/youtube_dl/extractor/xbef.py
index 1b4e88365..80c48c37d 100644
--- a/youtube_dl/extractor/xbef.py
+++ b/youtube_dl/extractor/xbef.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
@@ -23,10 +21,9 @@ class XBefIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         title = self._html_search_regex(
             r'<h1[^>]*>(.*?)</h1>', webpage, 'title')
 
diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py
index 53ed7ef5a..79ed6c744 100644
--- a/youtube_dl/extractor/xnxx.py
+++ b/youtube_dl/extractor/xnxx.py
@@ -1,10 +1,8 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
@@ -23,10 +21,7 @@ class XNXXIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        # Get webpage content
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._search_regex(r'flv_url=(.*?)&amp;',
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 38448e7c0..f9d98b83f 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -4,15 +4,17 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     parse_duration,
     str_to_int,
 )
 
 
 class XTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<videoid>[^/?&]+))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<id>[^/?&]+))'
     _TEST = {
         'url': 'http://www.xtube.com/watch.php?v=kVTUy_G222_',
         'md5': '092fbdd3cbe292c920ef6fc6a8a9cdab',
@@ -29,7 +31,7 @@ class XTubeIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
+        video_id = mobj.group('id')
         url = 'http://www.' + mobj.group('url')
 
         req = compat_urllib_request.Request(url)
diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 7e0044824..2a45dc574 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -3,15 +3,17 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
-    ExtractorError,
+)
+from ..utils import (
     clean_html,
+    ExtractorError,
 )
 
 
 class XVideosIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
+    _VALID_URL = r'https?://(?:www\.)?xvideos\.com/video(?P<id>[0-9]+)(?:.*)'
     _TEST = {
         'url': 'http://www.xvideos.com/video4588838/biker_takes_his_girl',
         'md5': '4b46ae6ea5e6e9086e714d883313c0c9',
@@ -24,37 +26,25 @@ class XVideosIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
-
         mobj = re.search(r'<h1 class="inlineError">(.+?)</h1>', webpage)
         if mobj:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, clean_html(mobj.group(1))), expected=True)
 
-        # Extract video URL
         video_url = compat_urllib_parse.unquote(
             self._search_regex(r'flv_url=(.+?)&', webpage, 'video URL'))
-
-        # Extract title
         video_title = self._html_search_regex(
             r'<title>(.*?)\s+-\s+XVID', webpage, 'title')
-
-        # Extract video thumbnail
         video_thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,
             'url': video_url,
-            'uploader': None,
-            'upload_date': None,
             'title': video_title,
             'ext': 'flv',
             'thumbnail': video_thumbnail,
-            'description': None,
             'age_limit': 18,
         }
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 0fdb12243..031226f27 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -6,11 +6,13 @@ import json
 import re
 
 from .common import InfoExtractor, SearchInfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
+)
+from ..utils import (
     clean_html,
+    ExtractorError,
     int_or_none,
 )
 
diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 7b621a9e3..894678a23 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -5,7 +5,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import compat_urllib_parse
+from ..compat import compat_urllib_parse
 
 
 class YnetIE(InfoExtractor):
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index d9c06a2ee..107c9ac36 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -6,10 +6,11 @@ import re
 import sys
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-
+)
+from ..utils import (
     ExtractorError,
     unescapeHTML,
     unified_strdate,
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 2bd264b30..e60505ace 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -4,8 +4,8 @@ import collections
 import io
 import zlib
 
+from .compat import compat_str
 from .utils import (
-    compat_str,
     ExtractorError,
     struct_unpack,
 )

From 7129bed51bc2452524c69874023ee4a0b351a046 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 12:35:13 +0100
Subject: [PATCH 197/974] [keek] Modernize and extract uploader

---
 youtube_dl/extractor/keek.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 5d679e88d..e94e2f8ad 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -6,29 +6,36 @@ from .common import InfoExtractor
 
 
 class KeekIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P<videoID>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P<id>\w+)'
     IE_NAME = 'keek'
     _TEST = {
         'url': 'https://www.keek.com/ytdl/keeks/NODfbab',
-        'file': 'NODfbab.mp4',
-        'md5': '9b0636f8c0f7614afa4ea5e4c6e57e83',
+        'md5': '09c5c109067536c1cec8bac8c21fea05',
         'info_dict': {
-            'uploader': 'ytdl',
+            'id': 'NODfbab',
+            'ext': 'mp4',
+            'uploader': 'youtube-dl project',
+            'uploader_id': 'ytdl',
             'title': 'test chars: "\'/\\\u00e4<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de .',
         },
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('videoID')
+        video_id = self._match_id(url)
 
         video_url = 'http://cdn.keek.com/keek/video/%s' % video_id
         thumbnail = 'http://cdn.keek.com/keek/thumbnail/%s/w100/h75' % video_id
         webpage = self._download_webpage(url, video_id)
 
-        uploader = self._html_search_regex(
-            r'<div class="user-name-and-bio">[\S\s]+?<h2>(?P<uploader>.+?)</h2>',
-            webpage, 'uploader', fatal=False)
+        raw_desc = self._html_search_meta('description', webpage)
+        if raw_desc:
+            uploader = self._html_search_regex(
+                r'Watch (.*?)\s+\(', raw_desc, 'uploader', fatal=False)
+            uploader_id = self._html_search_regex(
+                r'Watch .*?\(@(.+?)\)', raw_desc, 'uploader_id', fatal=False)
+        else:
+            uploader = None
+            uploader_id = None
 
         return {
             'id': video_id,
@@ -36,5 +43,6 @@ class KeekIE(InfoExtractor):
             'ext': 'mp4',
             'title': self._og_search_title(webpage),
             'thumbnail': thumbnail,
-            'uploader': uploader
+            'uploader': uploader,
+            'uploader_id': uploader_id,
         }

From 89e36657ccd382249c55701b00296b3ed78d29e8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 12:36:46 +0100
Subject: [PATCH 198/974] [keek] remove unused import

---
 youtube_dl/extractor/keek.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index e94e2f8ad..c0956ba09 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 

From a6620ac28df7ddbe39f09575d8e9f7fe73398a00 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 12:41:31 +0100
Subject: [PATCH 199/974] [orf] Modernize

---
 youtube_dl/extractor/orf.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 572a234ad..a6e722bf5 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -20,21 +20,23 @@ class ORFTVthekIE(InfoExtractor):
     _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics/.+?|program/[^/]+)/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://tvthek.orf.at/program/matinee-Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7317210/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319746/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319747',
-        'file': '7319747.mp4',
-        'md5': 'bd803c5d8c32d3c64a0ea4b4eeddf375',
-        'info_dict': {
-            'title': 'Was Sie schon immer über Klassik wissen wollten',
-            'description': 'md5:0ddf0d5f0060bd53f744edaa5c2e04a4',
-            'duration': 3508,
-            'upload_date': '20140105',
-        },
+        'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
+        'playlist': [{
+            'md5': '2942210346ed779588f428a92db88712',
+            'info_dict': {
+                'id': '8896777',
+                'ext': 'mp4',
+                'title': 'Aufgetischt: Mit der Steirischen Tafelrunde',
+                'description': 'md5:c1272f0245537812d4e36419c207b67d',
+                'duration': 2668,
+                'upload_date': '20141208',
+            },
+        }],
         'skip': 'Blocked outside of Austria',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
+        playlist_id = self._match_id(url)
         webpage = self._download_webpage(url, playlist_id)
 
         data_json = self._search_regex(
@@ -120,9 +122,7 @@ class ORFOE1IE(InfoExtractor):
     _VALID_URL = r'http://oe1\.orf\.at/programm/(?P<id>[0-9]+)'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        show_id = mobj.group('id')
-
+        show_id = self._match_id(url)
         data = self._download_json(
             'http://oe1.orf.at/programm/%s/konsole' % show_id,
             show_id

From 1557ed153c42176335f2ef069661ef2c98f16ce4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 13 Dec 2014 12:39:58 +0100
Subject: [PATCH 200/974] [test_unicode_literals] Import from test.helper

---
 test/test_unicode_literals.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index d3cba869b..19813e034 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -20,7 +20,7 @@ IGNORED_FILES = [
 ]
 
 
-from helper import assertRegexpMatches
+from test.helper import assertRegexpMatches
 
 
 class TestUnicodeLiterals(unittest.TestCase):

From 8085fc15ccb161d96eee607bf21dfdb36b890b20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Dec 2014 18:42:29 +0600
Subject: [PATCH 201/974] [adultswim] Improve segment duration extraction

---
 youtube_dl/extractor/adultswim.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 39e4ca296..502a9c25a 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -7,6 +7,8 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    xpath_text,
+    float_or_none,
 )
 
 
@@ -128,7 +130,8 @@ class AdultSwimIE(InfoExtractor):
                 segment_url, segment_title,
                 'Downloading segment information', 'Unable to download segment information')
 
-            segment_duration = idoc.find('.//trt').text.strip()
+            segment_duration = float_or_none(
+                xpath_text(idoc, './/trt', 'segment duration').strip())
 
             formats = []
             file_els = idoc.findall('.//files/file')

From 04c95441871775dfdaa0ea3eeca0a5d23f270f2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Dec 2014 18:47:34 +0600
Subject: [PATCH 202/974] [bbccouk] Fix vpid warning

---
 youtube_dl/extractor/bbccouk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 01c02d360..2d2f742ae 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -209,7 +209,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
         programme_id = self._search_regex(
-            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False)
+            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
         if programme_id:
             player = self._download_json(
                 'http://www.bbc.co.uk/iplayer/episode/%s.json' % group_id,

From 1fa174692aae06406c9524fec392a131f10c68fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Dec 2014 02:00:54 +0600
Subject: [PATCH 203/974] [bandcamp:album] Make path optional (Closes #4461)

---
 youtube_dl/extractor/bandcamp.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 9fb770cb1..bcb2821cd 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -106,7 +106,7 @@ class BandcampIE(InfoExtractor):
 
 class BandcampAlbumIE(InfoExtractor):
     IE_NAME = 'Bandcamp:album'
-    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+))'
+    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+))?'
 
     _TESTS = [{
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
@@ -141,6 +141,12 @@ class BandcampAlbumIE(InfoExtractor):
             'title': 'Hierophany of the Open Grave',
         },
         'playlist_mincount': 9,
+    }, {
+        'url': 'http://dotscale.bandcamp.com',
+        'info_dict': {
+            'title': 'Loom',
+        },
+        'playlist_mincount': 7,
     }]
 
     def _real_extract(self, url):

From 4a0132c570e29075e377c8b70d7f61018d1e4479 Mon Sep 17 00:00:00 2001
From: Mathias Rav <rav@cs.au.dk>
Date: Sat, 13 Dec 2014 21:58:52 +0100
Subject: [PATCH 204/974] [Restudy] Add new extractor for restudy.dk

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/restudy.py  | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/restudy.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 119ec2044..3ae7a8a52 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -316,6 +316,7 @@ from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
 from .redtube import RedTubeIE
+from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
 from .ringtv import RingTVIE
 from .ro220 import Ro220IE
diff --git a/youtube_dl/extractor/restudy.py b/youtube_dl/extractor/restudy.py
new file mode 100644
index 000000000..56a6c0f93
--- /dev/null
+++ b/youtube_dl/extractor/restudy.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RestudyIE(InfoExtractor):
+    _VALID_URL = r'https://www.restudy.dk/video/play/id/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://www.restudy.dk/video/play/id/1637',
+        # MD5 sum of first 10241 bytes of the video file, as reported by
+        # head -c 10241 Leiden-frosteffekt-1637.mp4 | md5sum
+        'md5': '4e755c4287f292a1fe5363834a683818',
+        'info_dict': {
+            'id': '1637',
+            'ext': 'mp4',
+            'title': 'Leiden-frosteffekt',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        xml_url = (
+            'https://www.restudy.dk/awsmedia/SmilDirectory/video_%s.xml'
+            % video_id)
+        xml = self._download_webpage(xml_url, video_id)
+
+        base = self._search_regex(
+            r'<meta base="([^"]+)', xml, 'meta base')
+        # TODO: Provide multiple video qualities instead of forcing highest
+        filename = self._search_regex(
+            r'<video src="mp4:([^"]+_high\.mp4)', xml, 'filename')
+        url = '%s%s' % (base, filename)
+        title = self._og_search_title(webpage)
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+            'protocol': 'rtmp',
+        }

From ac265bef1e685706c0be89237117e2df5f796ecb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Dec 2014 03:41:00 +0600
Subject: [PATCH 205/974] [restudy] Simplify and extract all formats

---
 youtube_dl/extractor/restudy.py | 37 ++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/restudy.py b/youtube_dl/extractor/restudy.py
index 56a6c0f93..b17c2bfc0 100644
--- a/youtube_dl/extractor/restudy.py
+++ b/youtube_dl/extractor/restudy.py
@@ -5,37 +5,36 @@ from .common import InfoExtractor
 
 
 class RestudyIE(InfoExtractor):
-    _VALID_URL = r'https://www.restudy.dk/video/play/id/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?restudy\.dk/video/play/id/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'https://www.restudy.dk/video/play/id/1637',
-        # MD5 sum of first 10241 bytes of the video file, as reported by
-        # head -c 10241 Leiden-frosteffekt-1637.mp4 | md5sum
-        'md5': '4e755c4287f292a1fe5363834a683818',
         'info_dict': {
             'id': '1637',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Leiden-frosteffekt',
+            'description': 'Denne video er et eksperiment med flydende kvælstof.',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        xml_url = (
-            'https://www.restudy.dk/awsmedia/SmilDirectory/video_%s.xml'
-            % video_id)
-        xml = self._download_webpage(xml_url, video_id)
 
-        base = self._search_regex(
-            r'<meta base="([^"]+)', xml, 'meta base')
-        # TODO: Provide multiple video qualities instead of forcing highest
-        filename = self._search_regex(
-            r'<video src="mp4:([^"]+_high\.mp4)', xml, 'filename')
-        url = '%s%s' % (base, filename)
-        title = self._og_search_title(webpage)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage).strip()
+        description = self._og_search_description(webpage).strip()
+
+        formats = self._extract_smil_formats(
+            'https://www.restudy.dk/awsmedia/SmilDirectory/video_%s.xml' % video_id,
+            video_id)
+
         return {
             'id': video_id,
             'title': title,
-            'url': url,
-            'protocol': 'rtmp',
+            'description': description,
+            'formats': formats,
         }

From 2564300e55ec30754cbc416aaa134cbb0da1a5f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Dec 2014 03:42:42 +0600
Subject: [PATCH 206/974] Credit @Mortal for restudy (#4463)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bfa00f91b..5d45e02fc 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -92,3 +92,4 @@ Tithen-Firion
 Zack Fernandes
 cryptonaut
 Adrian Kretz
+Mathias Rav

From da4d4191a9037ef439ef3f93f02935c67dfcee85 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:05:22 +0100
Subject: [PATCH 207/974] Merge branch 'master' of github.com:rg3/youtube-dl

---
 test/test_unicode_literals.py     | 2 +-
 youtube_dl/downloader/f4m.py      | 1 +
 youtube_dl/extractor/adultswim.py | 5 ++++-
 youtube_dl/extractor/bbccouk.py   | 2 +-
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index d3cba869b..19813e034 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -20,7 +20,7 @@ IGNORED_FILES = [
 ]
 
 
-from helper import assertRegexpMatches
+from test.helper import assertRegexpMatches
 
 
 class TestUnicodeLiterals(unittest.TestCase):
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 00f3a026c..ef3e0d5f4 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -233,6 +233,7 @@ class F4mFD(FileDownloader):
                 'continuedl': True,
                 'quiet': True,
                 'noprogress': True,
+                'ratelimit': self.params.get('ratelimit', None),
                 'test': self.params.get('test', False),
             }
         )
diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 39e4ca296..502a9c25a 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -7,6 +7,8 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    xpath_text,
+    float_or_none,
 )
 
 
@@ -128,7 +130,8 @@ class AdultSwimIE(InfoExtractor):
                 segment_url, segment_title,
                 'Downloading segment information', 'Unable to download segment information')
 
-            segment_duration = idoc.find('.//trt').text.strip()
+            segment_duration = float_or_none(
+                xpath_text(idoc, './/trt', 'segment duration').strip())
 
             formats = []
             file_els = idoc.findall('.//files/file')
diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 01c02d360..2d2f742ae 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -209,7 +209,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
         programme_id = self._search_regex(
-            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False)
+            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
         if programme_id:
             player = self._download_json(
                 'http://www.bbc.co.uk/iplayer/episode/%s.json' % group_id,

From 051c46256b47bae1bd0f07b5dad2b4ad041506d0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:13:48 +0100
Subject: [PATCH 208/974] release 2014.12.13

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index da959ea85..1e47c55a0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.7'
+__version__ = '2014.12.13'

From a23669220a6d45bfc01d5cf9b047eba42e772003 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:27:21 +0100
Subject: [PATCH 209/974] [utils] Make ssl work on Python 2.7.8

---
 youtube_dl/utils.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ec34dcef9..ac66f3de0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -393,8 +393,14 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
         context.options &= ~ssl.OP_NO_SSLv3  # Allow older, not-as-secure SSLv3
         if opts_no_check_certificate:
             context.verify_mode = ssl.CERT_NONE
-        return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
-    elif sys.version_info < (3, 2):
+        try:
+            return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+        except TypeError:
+            # Python 2.7.8
+            # (create_default_context present but HTTPSHandler has no context=)
+            pass
+
+    if sys.version_info < (3, 2):
         import httplib
 
         class HTTPSConnectionV3(httplib.HTTPSConnection):

From 2128b696b87ed1b44aac0ee31deb94b0b777f7db Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:44:50 +0100
Subject: [PATCH 210/974] [utils] Do not make an exception for SSLv3

SSLv3 is terminally vulnerable to POODLE; web browsers are currently deprecating/removing it.
Closes #4459, fixes #4294
---
 youtube_dl/utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ac66f3de0..5e92bcc71 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -390,7 +390,6 @@ def formatSeconds(secs):
 def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
         context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
-        context.options &= ~ssl.OP_NO_SSLv3  # Allow older, not-as-secure SSLv3
         if opts_no_check_certificate:
             context.verify_mode = ssl.CERT_NONE
         try:

From 4d144be8b0b93c052648e8137ffd961deee4516e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:50:04 +0100
Subject: [PATCH 211/974] [bandcamp:album] Do not match plain Bandcamp URLs
 (#4461)

The _VALID_URL 1fa174692aae06406c9524fec392a131f10c68fe is to broad, since it matches everything beginning with bandcamp.com.
---
 youtube_dl/extractor/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index bcb2821cd..b45d68a61 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -106,7 +106,7 @@ class BandcampIE(InfoExtractor):
 
 class BandcampAlbumIE(InfoExtractor):
     IE_NAME = 'Bandcamp:album'
-    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+))?'
+    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+)|/?(?:$|[?#]))'
 
     _TESTS = [{
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',

From 6f18cc9abc081c715596dc3616dd8362a427a962 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:51:57 +0100
Subject: [PATCH 212/974] release 2014.12.13.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1e47c55a0..e502affe7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.13'
+__version__ = '2014.12.13.1'

From bad5c1a3037e7630d599a002fa80790a446cfe77 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 00:13:07 +0100
Subject: [PATCH 213/974] [rtp] Also match e-id-less URLs (#4382)

---
 youtube_dl/extractor/rtp.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index 0bed1f100..54f67a52f 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -8,8 +8,8 @@ from ..utils import js_to_json
 
 
 class RTPIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/e(?P<id>[0-9]+)/?'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/(?P<id>[^/?#]+)/?'
+    _TESTS = [{
         'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
         'info_dict': {
             'id': '174042',
@@ -21,7 +21,10 @@ class RTPIE(InfoExtractor):
         'params': {
             'skip_download': True,  # RTMP download
         },
-    }
+    }, {
+        'url': 'http://www.rtp.pt/play/p831/a-quimica-das-coisas',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 4927de4f865eea988b92f37df40f3522c024f6e6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 00:13:17 +0100
Subject: [PATCH 214/974] release 2014.12.14

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e502affe7..605ba5cf4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.13.1'
+__version__ = '2014.12.14'

From 60620368d75346212e9eb3ec6e3e7f2c630a5b0e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 00:43:34 +0100
Subject: [PATCH 215/974] [youtube] Fix player ID detection

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 823d6aaf3..3319d6bff 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -478,7 +478,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
         id_m = re.match(
-            r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.(?P<ext>[a-z]+)$',
+            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.(?P<ext>[a-z]+)$',
             player_url)
         if not id_m:
             raise ExtractorError('Cannot identify player %r' % player_url)

From 408b5839b186ffaea09a67e584b8da33525ed967 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 14 Dec 2014 17:59:25 +0200
Subject: [PATCH 216/974] [yesjapan] Add new extractor (Closes #4466)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/yesjapan.py | 62 ++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 youtube_dl/extractor/yesjapan.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3ae7a8a52..caf8c04f1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -510,6 +510,7 @@ from .yahoo import (
     YahooIE,
     YahooSearchIE,
 )
+from .yesjapan import YesJapanIE
 from .ynet import YnetIE
 from .youjizz import YouJizzIE
 from .youku import YoukuIE
diff --git a/youtube_dl/extractor/yesjapan.py b/youtube_dl/extractor/yesjapan.py
new file mode 100644
index 000000000..fec1ad1ac
--- /dev/null
+++ b/youtube_dl/extractor/yesjapan.py
@@ -0,0 +1,62 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    HEADRequest,
+    get_element_by_attribute,
+    parse_iso8601,
+)
+
+
+class YesJapanIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?yesjapan\.com/video/(?P<slug>[A-Za-z0-9\-]*)_(?P<id>[A-Za-z0-9]+)\.html'
+    _TEST = {
+        'url': 'http://www.yesjapan.com/video/japanese-in-5-20-wa-and-ga-particle-usages_726497834.html',
+        'md5': 'f0be416314e5be21a12b499b330c21cf',
+        'info_dict': {
+            'id': '726497834',
+            'title': 'Japanese in 5! #20 - WA And GA Particle Usages',
+            'description': 'This should clear up some issues most students of Japanese encounter with WA and GA....',
+            'ext': 'mp4',
+            'timestamp': 1416391590,
+            'upload_date': '20141119',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._og_search_title(webpage)
+        video_url = self._og_search_video_url(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        timestamp = None
+        submit_info = get_element_by_attribute('class', 'pm-submit-data', webpage)
+        if submit_info:
+            timestamp = parse_iso8601(self._search_regex(
+                r'datetime="([^"]+)"', webpage, 'upload date', fatal=False, default=None))
+
+        # attempt to resolve the final URL in order to get a proper extension
+        redirect_req = HEADRequest(video_url)
+        req = self._request_webpage(
+            redirect_req, video_id, note='Resolving final URL', errnote='Could not resolve final URL', fatal=False)
+        if req:
+            video_url = req.geturl()
+
+        formats = [{
+            'format_id': 'sd',
+            'url': video_url,
+        }]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'timestamp': timestamp,
+            'thumbnail': thumbnail,
+        }

From 4a4fbfc9670cb3d2e5f0bc486c83ef1e6eb21ad3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 14 Dec 2014 18:03:05 +0200
Subject: [PATCH 217/974] [yesjapan] Look for datetime inside `submit_info`

Oops..
---
 youtube_dl/extractor/yesjapan.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yesjapan.py b/youtube_dl/extractor/yesjapan.py
index fec1ad1ac..112a6c030 100644
--- a/youtube_dl/extractor/yesjapan.py
+++ b/youtube_dl/extractor/yesjapan.py
@@ -38,7 +38,7 @@ class YesJapanIE(InfoExtractor):
         submit_info = get_element_by_attribute('class', 'pm-submit-data', webpage)
         if submit_info:
             timestamp = parse_iso8601(self._search_regex(
-                r'datetime="([^"]+)"', webpage, 'upload date', fatal=False, default=None))
+                r'datetime="([^"]+)"', submit_info, 'upload date', fatal=False, default=None))
 
         # attempt to resolve the final URL in order to get a proper extension
         redirect_req = HEADRequest(video_url)

From a0ab29f8a10e2a97f87182a85a6309ae61ec27b0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 21:55:57 +0100
Subject: [PATCH 218/974] Add offlinetest make target

---
 Makefile | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index d5b6e4307..2a32a6c0c 100644
--- a/Makefile
+++ b/Makefile
@@ -35,13 +35,27 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtu
 	install -d $(DESTDIR)$(SYSCONFDIR)/fish/completions
 	install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish
 
+codetest:
+	PYFLAKES_OUT=$$(pyflakes youtube_dl | grep -v youtube_dl/extractor/__init__.py); \
+	if test -n "$$PYFLAKES_OUT"; then \
+		echo "$$PYFLAKES_OUT"; \
+		exit 1; \
+	fi
+	pep8 . --ignore E501
+
 test:
 	#nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test
 	nosetests --verbose test
+	$(MAKE) codetest
+
+ot: offlinetest
+
+offlinetest: codetest
+	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles
 
 tar: youtube-dl.tar.gz
 
-.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion
+.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest
 
 pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish
 

From 6cbf345f28f5950fb274c70be12f5d91679bf787 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 21:56:12 +0100
Subject: [PATCH 219/974] Remove test/write_info_json

This is now covered by every single test_download testcase anyways :)
---
 test/test_write_info_json.py | 76 ------------------------------------
 1 file changed, 76 deletions(-)
 delete mode 100644 test/test_write_info_json.py

diff --git a/test/test_write_info_json.py b/test/test_write_info_json.py
deleted file mode 100644
index 0396ef262..000000000
--- a/test/test_write_info_json.py
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-from __future__ import unicode_literals
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from test.helper import get_params
-
-
-import io
-import json
-
-import youtube_dl.YoutubeDL
-import youtube_dl.extractor
-
-
-class YoutubeDL(youtube_dl.YoutubeDL):
-    def __init__(self, *args, **kwargs):
-        super(YoutubeDL, self).__init__(*args, **kwargs)
-        self.to_stderr = self.to_screen
-
-params = get_params({
-    'writeinfojson': True,
-    'skip_download': True,
-    'writedescription': True,
-})
-
-
-TEST_ID = 'BaW_jenozKc'
-INFO_JSON_FILE = TEST_ID + '.info.json'
-DESCRIPTION_FILE = TEST_ID + '.mp4.description'
-EXPECTED_DESCRIPTION = '''test chars:  "'/\ä↭𝕐
-test URL: https://github.com/rg3/youtube-dl/issues/1892
-
-This is a test video for youtube-dl.
-
-For more information, contact phihag@phihag.de .'''
-
-
-class TestInfoJSON(unittest.TestCase):
-    def setUp(self):
-        # Clear old files
-        self.tearDown()
-
-    def test_info_json(self):
-        ie = youtube_dl.extractor.YoutubeIE()
-        ydl = YoutubeDL(params)
-        ydl.add_info_extractor(ie)
-        ydl.download([TEST_ID])
-        self.assertTrue(os.path.exists(INFO_JSON_FILE))
-        with io.open(INFO_JSON_FILE, 'r', encoding='utf-8') as jsonf:
-            jd = json.load(jsonf)
-        self.assertEqual(jd['upload_date'], '20121002')
-        self.assertEqual(jd['description'], EXPECTED_DESCRIPTION)
-        self.assertEqual(jd['id'], TEST_ID)
-        self.assertEqual(jd['extractor'], 'youtube')
-        self.assertEqual(jd['title'], '''youtube-dl test video "'/\ä↭𝕐''')
-        self.assertEqual(jd['uploader'], 'Philipp Hagemeister')
-
-        self.assertTrue(os.path.exists(DESCRIPTION_FILE))
-        with io.open(DESCRIPTION_FILE, 'r', encoding='utf-8') as descf:
-            descr = descf.read()
-        self.assertEqual(descr, EXPECTED_DESCRIPTION)
-
-    def tearDown(self):
-        if os.path.exists(INFO_JSON_FILE):
-            os.remove(INFO_JSON_FILE)
-        if os.path.exists(DESCRIPTION_FILE):
-            os.remove(DESCRIPTION_FILE)
-
-if __name__ == '__main__':
-    unittest.main()

From cae97f6521a846392e665d8743d4c000fb7d7173 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 21:59:59 +0100
Subject: [PATCH 220/974] Improve and test ffmpeg version detection

---
 test/test_utils.py  | 38 +++++++++++++++++++++++++-------------
 youtube_dl/utils.py | 17 ++++++++++++-----
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index d42df6d96..dd49a6d17 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -16,39 +16,40 @@ import json
 import xml.etree.ElementTree
 
 from youtube_dl.utils import (
+    args_to_str,
     clean_html,
     DateRange,
+    detect_exe_version,
     encodeFilename,
+    escape_rfc3986,
+    escape_url,
     find_xpath_attr,
     fix_xml_ampersands,
-    orderedSet,
-    OnDemandPagedList,
     InAdvancePagedList,
+    intlist_to_bytes,
+    js_to_json,
+    limit_length,
+    OnDemandPagedList,
+    orderedSet,
     parse_duration,
+    parse_filesize,
+    parse_iso8601,
     read_batch_urls,
     sanitize_filename,
     shell_quote,
     smuggle_url,
     str_to_int,
+    strip_jsonp,
     struct_unpack,
     timeconvert,
     unescapeHTML,
     unified_strdate,
     unsmuggle_url,
+    uppercase_escape,
     url_basename,
     urlencode_postdata,
-    xpath_with_ns,
-    parse_iso8601,
-    strip_jsonp,
-    uppercase_escape,
-    limit_length,
-    escape_rfc3986,
-    escape_url,
-    js_to_json,
-    intlist_to_bytes,
-    args_to_str,
-    parse_filesize,
     version_tuple,
+    xpath_with_ns,
 )
 
 
@@ -390,5 +391,16 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(version_tuple('10.23.344'), (10, 23, 344))
         self.assertEqual(version_tuple('10.1-6'), (10, 1, 6))  # avconv style
 
+    def test_detect_exe_version(self):
+        self.assertEqual(detect_exe_version('''ffmpeg version 1.2.1
+built on May 27 2013 08:37:26 with gcc 4.7 (Debian 4.7.3-4)
+configuration: --prefix=/usr --extra-'''), '1.2.1')
+        self.assertEqual(detect_exe_version('''ffmpeg version N-63176-g1fb4685
+built on May 15 2014 22:09:06 with gcc 4.8.2 (GCC)'''), 'N-63176-g1fb4685')
+        self.assertEqual(detect_exe_version('''X server found. dri2 connection failed!
+Trying to open render node...
+Success at /dev/dri/renderD128.
+ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5e92bcc71..f9938616d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1262,18 +1262,25 @@ def check_executable(exe, args=[]):
 
 
 def get_exe_version(exe, args=['--version'],
-                    version_re=r'version\s+([0-9._-a-zA-Z]+)',
-                    unrecognized='present'):
+                    version_re=None, unrecognized='present'):
     """ Returns the version of the specified executable,
     or False if the executable is not present """
     try:
-        out, err = subprocess.Popen(
+        out, _ = subprocess.Popen(
             [exe] + args,
             stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
     except OSError:
         return False
-    firstline = out.partition(b'\n')[0].decode('ascii', 'ignore')
-    m = re.search(version_re, firstline)
+    if isinstance(out, bytes):  # Python 2.x
+        out = out.decode('ascii', 'ignore')
+    return detect_exe_version(out, version_re, unrecognized)
+
+
+def detect_exe_version(output, version_re=None, unrecognized='present'):
+    assert isinstance(output, compat_str)
+    if version_re is None:
+        version_re = r'version\s+([-0-9._a-zA-Z]+)'
+    m = re.search(version_re, output)
     if m:
         return m.group(1)
     else:

From 39f594d660cd86f04905d3eead91c5e2f16b0e66 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 00:59:23 +0100
Subject: [PATCH 221/974] [Makefile] Ensure that offline test really is offline

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2a32a6c0c..6fb820c2a 100644
--- a/Makefile
+++ b/Makefile
@@ -51,7 +51,7 @@ test:
 ot: offlinetest
 
 offlinetest: codetest
-	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles
+	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles --exclude test_write_annotations
 
 tar: youtube-dl.tar.gz
 

From 4f026fafbc6cc4eac10a5f89b9375b44d64083c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 01:06:25 +0100
Subject: [PATCH 222/974] [YoutubeDL] Make postprocessors declarative

Instead of having to configure PPs in code, this allows us and embedding programs not to worry about imports or finer details, similarly to how we handle IEs.
---
 README.md                            | 28 +++++++++++
 youtube_dl/YoutubeDL.py              | 19 +++++++-
 youtube_dl/__init__.py               | 71 +++++++++++++++-------------
 youtube_dl/postprocessor/__init__.py |  9 +++-
 youtube_dl/postprocessor/ffmpeg.py   |  4 +-
 5 files changed, 93 insertions(+), 38 deletions(-)

diff --git a/README.md b/README.md
index f10f06ee8..edfcfc223 100644
--- a/README.md
+++ b/README.md
@@ -537,6 +537,34 @@ From a Python program, you can embed youtube-dl in a more powerful fashion, like
 
 Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
+Here's a more complete example of a program that only outputs errors, and downloads/converts the video as mp3:
+
+
+    import youtube_dl
+
+
+    class MyLogger(object):
+        def debug(self, msg):
+            pass
+
+        def warning(self, msg):
+            pass
+
+        def error(self, msg):
+            print(msg)
+
+    ydl_opts = {
+        'format': 'bestaudio/best',
+        'postprocessors': [{
+            'key': 'FFmpegExtractAudio',
+            'preferredcodec': 'mp3',
+            'preferredquality': '64',
+        }],
+        'logger': MyLogger(),
+    }
+    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
+        ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+
 # BUGS
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode.
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 578c8daf2..6acfd8cf9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -27,6 +27,7 @@ from .compat import (
     compat_cookiejar,
     compat_expanduser,
     compat_http_client,
+    compat_kwargs,
     compat_str,
     compat_urllib_error,
     compat_urllib_request,
@@ -67,7 +68,11 @@ from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
-from .postprocessor import FFmpegMergerPP, FFmpegPostProcessor
+from .postprocessor import (
+    FFmpegMergerPP,
+    FFmpegPostProcessor,
+    get_postprocessor,
+)
 from .version import __version__
 
 
@@ -176,6 +181,11 @@ class YoutubeDL(object):
     extract_flat:      Do not resolve URLs, return the immediate result.
                        Pass in 'in_playlist' to only show this behavior for
                        playlist items.
+    postprocessors:    A list of dictionaries, each with an entry
+                       key:  The name of the postprocessor. See
+                             youtube_dl/postprocessor/__init__.py for a list.
+                       as well as any further keyword arguments for the
+                       postprocessor.
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
@@ -256,6 +266,13 @@ class YoutubeDL(object):
             self.print_debug_header()
             self.add_default_info_extractors()
 
+        for pp_def_raw in self.params.get('postprocessors', []):
+            pp_class = get_postprocessor(pp_def_raw['key'])
+            pp_def = dict(pp_def_raw)
+            del pp_def['key']
+            pp = pp_class(self, **compat_kwargs(pp_def))
+            self.add_post_processor(pp)
+
     def warn_if_short_id(self, argv):
         # short YouTube ID starting with dash?
         idxs = [
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 70c4f25b1..e79320323 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -40,16 +40,6 @@ from .downloader import (
 )
 from .extractor import gen_extractors
 from .YoutubeDL import YoutubeDL
-from .postprocessor import (
-    AtomicParsleyPP,
-    FFmpegAudioFixPP,
-    FFmpegMetadataPP,
-    FFmpegVideoConvertor,
-    FFmpegExtractAudioPP,
-    FFmpegEmbedSubtitlePP,
-    XAttrMetadataPP,
-    ExecAfterDownloadPP,
-)
 
 
 def _real_main(argv=None):
@@ -212,6 +202,43 @@ def _real_main(argv=None):
     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
     download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
+    # PostProcessors
+    postprocessors = []
+    # Add the metadata pp first, the other pps will copy it
+    if opts.addmetadata:
+        postprocessors.append({'key': 'FFmpegMetadata'})
+    if opts.extractaudio:
+        postprocessors.append({
+            'key': 'FFmpegExtractAudio',
+            'preferredcodec': opts.audioformat,
+            'preferredquality': opts.audioquality,
+            'nopostoverwrites': opts.nopostoverwrites,
+        })
+    if opts.recodevideo:
+        postprocessors.append({
+            'key': 'FFmpegVideoConvertor',
+            'preferedformat': opts.recodevideo,
+        })
+    if opts.embedsubtitles:
+        postprocessors.append({
+            'key': 'FFmpegEmbedSubtitle',
+            'subtitlesformat': opts.subtitlesformat,
+        })
+    if opts.xattrs:
+        postprocessors.append({'key': 'XAttrMetadata'})
+    if opts.embedthumbnail:
+        if not opts.addmetadata:
+            postprocessors.append({'key': 'FFmpegAudioFix'})
+        postprocessors.append({'key': 'AtomicParsley'})
+    # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
+    # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
+    if opts.exec_cmd:
+        postprocessors.append({
+            'key': 'ExecAfterDownload',
+            'verboseOutput': opts.verbose,
+            'exec_cmd': opts.exec_cmd,
+        })
+
     ydl_opts = {
         'usenetrc': opts.usenetrc,
         'username': opts.username,
@@ -297,32 +324,10 @@ def _real_main(argv=None):
         'encoding': opts.encoding,
         'exec_cmd': opts.exec_cmd,
         'extract_flat': opts.extract_flat,
+        'postprocessors': postprocessors,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
-        # PostProcessors
-        # Add the metadata pp first, the other pps will copy it
-        if opts.addmetadata:
-            ydl.add_post_processor(FFmpegMetadataPP())
-        if opts.extractaudio:
-            ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
-        if opts.recodevideo:
-            ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
-        if opts.embedsubtitles:
-            ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
-        if opts.xattrs:
-            ydl.add_post_processor(XAttrMetadataPP())
-        if opts.embedthumbnail:
-            if not opts.addmetadata:
-                ydl.add_post_processor(FFmpegAudioFixPP())
-            ydl.add_post_processor(AtomicParsleyPP())
-
-        # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
-        # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
-        if opts.exec_cmd:
-            ydl.add_post_processor(ExecAfterDownloadPP(
-                verboseOutput=opts.verbose, exec_cmd=opts.exec_cmd))
-
         # Update version
         if opts.update_self:
             update_self(ydl.to_screen, opts.verbose)
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index fb367ebe4..7f505b58e 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -8,11 +8,16 @@ from .ffmpeg import (
     FFmpegExtractAudioPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
-    FFmpegVideoConvertor,
+    FFmpegVideoConvertorPP,
 )
 from .xattrpp import XAttrMetadataPP
 from .execafterdownload import ExecAfterDownloadPP
 
+
+def get_postprocessor(key):
+    return globals()[key + 'PP']
+
+
 __all__ = [
     'AtomicParsleyPP',
     'ExecAfterDownloadPP',
@@ -22,6 +27,6 @@ __all__ = [
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
     'FFmpegPostProcessor',
-    'FFmpegVideoConvertor',
+    'FFmpegVideoConvertorPP',
     'XAttrMetadataPP',
 ]
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 965ded4c1..048525efc 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -236,9 +236,9 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         return self._nopostoverwrites, information
 
 
-class FFmpegVideoConvertor(FFmpegPostProcessor):
+class FFmpegVideoConvertorPP(FFmpegPostProcessor):
     def __init__(self, downloader=None, preferedformat=None):
-        super(FFmpegVideoConvertor, self).__init__(downloader)
+        super(FFmpegVideoConvertorPP, self).__init__(downloader)
         self._preferedformat = preferedformat
 
     def run(self, information):

From 71b640cc5b2f15a6913a720b589bdd3ed318c154 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 01:26:18 +0100
Subject: [PATCH 223/974] [YoutubeDL] Add declarative version of progress hooks

---
 README.md                       |  9 ++++++++-
 youtube_dl/YoutubeDL.py         | 24 ++++++++++++++++++++++--
 youtube_dl/downloader/common.py | 17 ++---------------
 3 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index edfcfc223..bd35a8214 100644
--- a/README.md
+++ b/README.md
@@ -537,7 +537,7 @@ From a Python program, you can embed youtube-dl in a more powerful fashion, like
 
 Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
-Here's a more complete example of a program that only outputs errors, and downloads/converts the video as mp3:
+Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
 
     import youtube_dl
@@ -553,6 +553,12 @@ Here's a more complete example of a program that only outputs errors, and downlo
         def error(self, msg):
             print(msg)
 
+
+    def my_hook(d):
+        if d['status'] == 'finished':
+            print('Done downloading, now converting ...')
+
+
     ydl_opts = {
         'format': 'bestaudio/best',
         'postprocessors': [{
@@ -561,6 +567,7 @@ Here's a more complete example of a program that only outputs errors, and downlo
             'preferredquality': '64',
         }],
         'logger': MyLogger(),
+        'progress_hooks': [my_hook],
     }
     with youtube_dl.YoutubeDL(ydl_opts) as ydl:
         ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6acfd8cf9..bea1ef83c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -182,10 +182,27 @@ class YoutubeDL(object):
                        Pass in 'in_playlist' to only show this behavior for
                        playlist items.
     postprocessors:    A list of dictionaries, each with an entry
-                       key:  The name of the postprocessor. See
-                             youtube_dl/postprocessor/__init__.py for a list.
+                       * key:  The name of the postprocessor. See
+                               youtube_dl/postprocessor/__init__.py for a list.
                        as well as any further keyword arguments for the
                        postprocessor.
+    progress_hooks:    A list of functions that get called on download
+                       progress, with a dictionary with the entries
+                       * filename: The final filename
+                       * status: One of "downloading" and "finished"
+
+                       The dict may also have some of the following entries:
+
+                       * downloaded_bytes: Bytes on disk
+                       * total_bytes: Size of the whole file, None if unknown
+                       * tmpfilename: The filename we're currently writing to
+                       * eta: The estimated time in seconds, None if unknown
+                       * speed: The download speed in bytes/second, None if
+                                unknown
+
+                       Progress hooks are guaranteed to be called at least once
+                       (with status "finished") if the download is successful.
+
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
@@ -273,6 +290,9 @@ class YoutubeDL(object):
             pp = pp_class(self, **compat_kwargs(pp_def))
             self.add_post_processor(pp)
 
+        for ph in self.params.get('progress_hooks', []):
+            self.add_progress_hook(ph)
+
     def warn_if_short_id(self, argv):
         # short YouTube ID starting with dash?
         idxs = [
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 8181bca09..d3e0b0110 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -305,19 +305,6 @@ class FileDownloader(object):
             ph(status)
 
     def add_progress_hook(self, ph):
-        """ ph gets called on download progress, with a dictionary with the entries
-        * filename: The final filename
-        * status: One of "downloading" and "finished"
-
-        It can also have some of the following entries:
-
-        * downloaded_bytes: Bytes on disks
-        * total_bytes: Total bytes, None if unknown
-        * tmpfilename: The filename we're currently writing to
-        * eta: The estimated time in seconds, None if unknown
-        * speed: The download speed in bytes/second, None if unknown
-
-        Hooks are guaranteed to be called at least once (with status "finished")
-        if the download is successful.
-        """
+        # See YoutubeDl.py (search for progress_hooks) for a description of
+        # this interface
         self._progress_hooks.append(ph)

From 8acb83d9937211abe7194161be863194ae445d95 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 01:34:39 +0100
Subject: [PATCH 224/974] [README] Make example audio sound not that horrible
 ;)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index bd35a8214..843e01c87 100644
--- a/README.md
+++ b/README.md
@@ -564,7 +564,7 @@ Here's a more complete example of a program that outputs only errors (and a shor
         'postprocessors': [{
             'key': 'FFmpegExtractAudio',
             'preferredcodec': 'mp3',
-            'preferredquality': '64',
+            'preferredquality': '192',
         }],
         'logger': MyLogger(),
         'progress_hooks': [my_hook],

From e97a534f13ef9bf3d8e9007b80344ca43b8f0a20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 01:36:46 +0100
Subject: [PATCH 225/974] release 2014.12.15

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 605ba5cf4..4d3adbf61 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.14'
+__version__ = '2014.12.15'

From 1bbe317508fa151336034eda4809d0fea1684667 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 15 Dec 2014 03:31:54 +0200
Subject: [PATCH 226/974] [mooshare] Modernize

---
 youtube_dl/extractor/mooshare.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 9f2853fa3..4c5a4e673 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -64,8 +64,7 @@ class MooshareIE(InfoExtractor):
             'http://mooshare.biz/%s' % video_id, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
-        self.to_screen('%s: Waiting for timeout' % video_id)
-        time.sleep(5)
+        self._sleep(5, video_id)
 
         video_page = self._download_webpage(request, video_id, 'Downloading video page')
 

From 6f9b54933f1380423258fa7fe478e002ca97027b Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 15 Dec 2014 03:32:17 +0200
Subject: [PATCH 227/974] [streamcloud] Modernize

---
 youtube_dl/extractor/streamcloud.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index 38176498d..fd0cd3c0c 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -40,8 +40,7 @@ class StreamcloudIE(InfoExtractor):
             ''', orig_webpage)
         post = compat_urllib_parse.urlencode(fields)
 
-        self.to_screen('%s: Waiting for timeout' % video_id)
-        time.sleep(12)
+        self._sleep(12, video_id)
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
         }

From 3013bbb27d79ba22dc432b1081acccef96b54931 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 15 Dec 2014 08:24:50 +0100
Subject: [PATCH 228/974] Remove unused imports

---
 youtube_dl/extractor/mooshare.py    | 1 -
 youtube_dl/extractor/streamcloud.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 4c5a4e673..7603af5e2 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import time
 
 from .common import InfoExtractor
 from ..compat import (
diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index fd0cd3c0c..d4e134015 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import time
 
 from .common import InfoExtractor
 from ..compat import (

From 99a0f9824a2b6890998d5d668213974907f9c2de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 11:11:52 +0100
Subject: [PATCH 229/974] [README] Highlight code examples

---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 843e01c87..80ef3dcca 100644
--- a/README.md
+++ b/README.md
@@ -529,17 +529,19 @@ youtube-dl makes the best effort to be a good command-line program, and thus sho
 
 From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
 
+    ```python
     import youtube_dl
 
     ydl_opts = {}
     with youtube_dl.YoutubeDL(ydl_opts) as ydl:
         ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+    ```
 
 Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
-
+    ```python
     import youtube_dl
 
 
@@ -571,6 +573,7 @@ Here's a more complete example of a program that outputs only errors (and a shor
     }
     with youtube_dl.YoutubeDL(ydl_opts) as ydl:
         ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+    ```
 
 # BUGS
 

From caff59499c17f1701ca9a6c654d001d70456b210 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 11:14:06 +0100
Subject: [PATCH 230/974] [README] Fix code rendering

---
 README.md | 62 +++++++++++++++++++++++++++----------------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/README.md b/README.md
index 80ef3dcca..ecc7302c8 100644
--- a/README.md
+++ b/README.md
@@ -529,51 +529,51 @@ youtube-dl makes the best effort to be a good command-line program, and thus sho
 
 From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
 
-    ```python
-    import youtube_dl
+```python
+import youtube_dl
 
-    ydl_opts = {}
-    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
-        ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
-    ```
+ydl_opts = {}
+with youtube_dl.YoutubeDL(ydl_opts) as ydl:
+    ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+```
 
 Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
-    ```python
-    import youtube_dl
+```python
+import youtube_dl
 
 
-    class MyLogger(object):
-        def debug(self, msg):
-            pass
+class MyLogger(object):
+    def debug(self, msg):
+        pass
 
-        def warning(self, msg):
-            pass
+    def warning(self, msg):
+        pass
 
-        def error(self, msg):
-            print(msg)
+    def error(self, msg):
+        print(msg)
 
 
-    def my_hook(d):
-        if d['status'] == 'finished':
-            print('Done downloading, now converting ...')
+def my_hook(d):
+    if d['status'] == 'finished':
+        print('Done downloading, now converting ...')
 
 
-    ydl_opts = {
-        'format': 'bestaudio/best',
-        'postprocessors': [{
-            'key': 'FFmpegExtractAudio',
-            'preferredcodec': 'mp3',
-            'preferredquality': '192',
-        }],
-        'logger': MyLogger(),
-        'progress_hooks': [my_hook],
-    }
-    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
-        ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
-    ```
+ydl_opts = {
+    'format': 'bestaudio/best',
+    'postprocessors': [{
+        'key': 'FFmpegExtractAudio',
+        'preferredcodec': 'mp3',
+        'preferredquality': '192',
+    }],
+    'logger': MyLogger(),
+    'progress_hooks': [my_hook],
+}
+with youtube_dl.YoutubeDL(ydl_opts) as ydl:
+    ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+```
 
 # BUGS
 

From 4b7df0d30cfe62ab25f462974c2ed0fc5a82eb3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 15 Dec 2014 19:19:15 +0100
Subject: [PATCH 231/974] [youtube:playlist] Work around buggy playlists (fixes
 #4449)

They show a "Load more" button, but they don't have more videos.
The continuation url in the json file was a link to itself, so we ended up in an infinite loop.
---
 youtube_dl/extractor/youtube.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3319d6bff..aaa07b5eb 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1128,6 +1128,13 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'info_dict': {
             'title': 'JODA7',
         }
+    }, {
+        'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
+        'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
+        'info_dict': {
+                'title': 'Uploads from Interstellar Movie',
+        },
+        'playlist_mincout': 21,
     }]
 
     def _real_initialize(self):
@@ -1212,6 +1219,10 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                 'Downloading page #%s' % page_num,
                 transform_source=uppercase_escape)
             content_html = more['content_html']
+            if not content_html.strip():
+                # Some webpages show a "Load more" button but they don't
+                # have more videos
+                break
             more_widget_html = more['load_more_widget_html']
 
         playlist_title = self._html_search_regex(

From d86007873ef721819241ecd84258e9cce02ec961 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 00:22:23 +0100
Subject: [PATCH 232/974] [YoutubeDL] Document where details for format can be
 found

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bea1ef83c..71c01facb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -121,7 +121,7 @@ class YoutubeDL(object):
     dump_single_json:  Force printing the info_dict of the whole playlist
                        (or video) as a single JSON line.
     simulate:          Do not download the video files.
-    format:            Video format code.
+    format:            Video format code. See options.py for more information.
     format_limit:      Highest quality format to try.
     outtmpl:           Template for output names.
     restrictfilenames: Do not allow "&" and spaces in file names

From ccbfaa83b032587750cf466802d3f9ad162253fe Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 00:22:38 +0100
Subject: [PATCH 233/974] [devscripts/make_contributing] Switch to optparse
 (Fixes #4483)

---
 devscripts/make_contributing.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py
index 5fa8cf851..5e454a429 100755
--- a/devscripts/make_contributing.py
+++ b/devscripts/make_contributing.py
@@ -1,20 +1,20 @@
 #!/usr/bin/env python
 from __future__ import unicode_literals
 
-import argparse
 import io
+import optparse
 import re
 
 
 def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'INFILE', help='README.md file name to read from')
-    parser.add_argument(
-        'OUTFILE', help='CONTRIBUTING.md file name to write to')
-    args = parser.parse_args()
+    parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
+    options, args = parser.parse_args()
+    if len(args) != 2:
+        parser.error('Expected an input and an output filename')
 
-    with io.open(args.INFILE, encoding='utf-8') as inf:
+    infile, outfile = args
+
+    with io.open(infile, encoding='utf-8') as inf:
         readme = inf.read()
 
     bug_text = re.search(
@@ -25,7 +25,7 @@ def main():
 
     out = bug_text + dev_text
 
-    with io.open(args.OUTFILE, 'w', encoding='utf-8') as outf:
+    with io.open(outfile, 'w', encoding='utf-8') as outf:
         outf.write(out)
 
 if __name__ == '__main__':

From 3940450878480cf12a12072a03db2c641c63c250 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 00:24:30 +0100
Subject: [PATCH 234/974] release 2014.12.16

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4d3adbf61..fb1bcdee1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.15'
+__version__ = '2014.12.16'

From 734ea11e3c57ca4df7d9cfc475b1b99b56c18034 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 00:37:42 +0100
Subject: [PATCH 235/974] Drop hash character in downloader output (#4484)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 71c01facb..8a6bcd51c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -712,7 +712,7 @@ class YoutubeDL(object):
                 entries = entries[::-1]
 
             for i, entry in enumerate(entries, 1):
-                self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries))
+                self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
                 extra = {
                     'n_entries': n_entries,
                     'playlist': playlist,

From 0647084f394ddbe440b012bdf30e1a4e00dcbebe Mon Sep 17 00:00:00 2001
From: Tailszefox <tails@tailszefox.no-ip.com>
Date: Tue, 16 Dec 2014 15:46:04 +0100
Subject: [PATCH 236/974] [Allocine] Support for more URLs

---
 youtube_dl/extractor/allocine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 623aeaf34..023e06fc6 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class AllocineIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?allocine\.fr/(?P<typ>article|video|film)/(fichearticle_gen_carticle=|player_gen_cmedia=|fichefilm_gen_cfilm=)(?P<id>[0-9]+)(?:\.html)?'
+    _VALID_URL = r'https?://(?:www\.)?allocine\.fr/(?P<typ>article|video|film)/(fichearticle_gen_carticle=|player_gen_cmedia=|fichefilm_gen_cfilm=|video-)(?P<id>[0-9]+)(?:\.html)?'
 
     _TESTS = [{
         'url': 'http://www.allocine.fr/article/fichearticle_gen_carticle=18635087.html',

From bb18d787b5c0ded4865090e8022338db063520b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Dec 2014 15:48:01 +0100
Subject: [PATCH 237/974] [aljazeera] Add extractor (closes #4487)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/aljazeera.py | 32 +++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 youtube_dl/extractor/aljazeera.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index caf8c04f1..b091505db 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -5,6 +5,7 @@ from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
 from .adultswim import AdultSwimIE
 from .aftonbladet import AftonbladetIE
+from .aljazeera import AlJazeeraIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
 from .aol import AolIE
diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
new file mode 100644
index 000000000..ace5ec19d
--- /dev/null
+++ b/youtube_dl/extractor/aljazeera.py
@@ -0,0 +1,32 @@
+from .common import InfoExtractor
+
+
+class AlJazeeraIE(InfoExtractor):
+    _VALID_URL = r'http://www\.aljazeera\.com/programmes/.*?/(?P<id>[^/]+)\.html'
+
+    _TEST = {
+        'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
+        'info_dict': {
+            'id': '3792260579001',
+            'ext': 'mp4',
+            'title': 'The Slum - Episode 1: Deliverance',
+            'description': 'As a birth attendant advocating for family planning, Remy is on the frontline of Tondo\'s battle with overcrowding.',
+            'uploader': 'Al Jazeera English',
+        },
+        'add_ie': ['Brightcove'],
+    }
+
+    def _real_extract(self, url):
+        program_name = self._match_id(url)
+        webpage = self._download_webpage(url, program_name)
+        brightcove_id = self._search_regex(
+            r'RenderPagesVideo\(\'(.+?)\'', webpage, 'brightcove id')
+
+        return {
+            '_type': 'url',
+            'url':
+                'http://c.brightcove.com/services/viewer/federated_f9?'
+                '&playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
+                '&%40videoPlayer={0}'.format(brightcove_id),
+            'ie_key': 'Brightcove',
+        }

From 84d84211acba32a2a9bfb8729eeffd05cd556918 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 15:59:31 +0100
Subject: [PATCH 238/974] [youtube:feeds] (Fixes #4486)

---
 youtube_dl/extractor/youtube.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index aaa07b5eb..100ecbc0a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1566,9 +1566,11 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
         feed_entries = []
         paging = 0
         for i in itertools.count(1):
-            info = self._download_json(self._FEED_TEMPLATE % paging,
-                                       '%s feed' % self._FEED_NAME,
-                                       'Downloading page %s' % i)
+            info = self._download_json(
+                self._FEED_TEMPLATE % paging,
+                '%s feed' % self._FEED_NAME,
+                'Downloading page %s' % i,
+                transform_source=uppercase_escape)
             feed_html = info.get('feed_html') or info.get('content_html')
             load_more_widget_html = info.get('load_more_widget_html') or feed_html
             m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)

From 9209fe38784dbb4d4c045537919f3dc50e02227a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Dec 2014 21:03:10 +0600
Subject: [PATCH 239/974] [allocine] Add test for new URL format

---
 youtube_dl/extractor/allocine.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 023e06fc6..7d65b8193 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -44,6 +44,9 @@ class AllocineIE(InfoExtractor):
             'description': 'md5:71742e3a74b0d692c7fce0dd2017a4ac',
             'thumbnail': 're:http://.*\.jpg',
         },
+    }, {
+        'url': 'http://www.allocine.fr/video/video-19550147/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From ac78b5e97be3827c1aa7e8dfa4a9f8f1a9cb8122 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 16:03:57 +0100
Subject: [PATCH 240/974] release 2014.12.16.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index fb1bcdee1..dacc47a78 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.16'
+__version__ = '2014.12.16.1'

From b50e3bc67fd864bbb14ed4ccf3f0d29e1ff3e1f0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 16:33:23 +0100
Subject: [PATCH 241/974] [README] Add table of contents (Closes #4458)

---
 README.md                     | 14 ++++++++++++--
 devscripts/prepare_manpage.py | 15 +++++++++++++--
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index ecc7302c8..b4afe17d2 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,15 @@
 youtube-dl - download videos from youtube.com or other video platforms
 
-# SYNOPSIS
-**youtube-dl** [OPTIONS] URL [URL...]
+- [INSTALLATION](#installation)
+- [DESCRIPTION](#description)
+- [OPTIONS](#options)
+- [CONFIGURATION](#configuration)
+- [OUTPUT TEMPLATE](#output-template)
+- [VIDEO SELECTION](#video-selection)
+- [FAQ](#faq)
+- [DEVELOPER INSTRUCTIONS](#developer-instructions)
+- [BUGS](#bugs)
+- [COPYRIGHT](#copyright)
 
 # INSTALLATION
 
@@ -34,6 +42,8 @@ YouTube.com and a few more sites. It requires the Python interpreter, version
 your Unix box, on Windows or on Mac OS X. It is released to the public domain,
 which means you can modify it, redistribute it or use it however you like.
 
+    youtube-dl [OPTIONS] URL [URL...]
+
 # OPTIONS
     -h, --help                       print this help text and exit
     --version                        print program version and exit
diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index f66bebfea..7ece37754 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -11,8 +11,19 @@ README_FILE = os.path.join(ROOT_DIR, 'README.md')
 with io.open(README_FILE, encoding='utf-8') as f:
     readme = f.read()
 
-PREFIX = '%YOUTUBE-DL(1)\n\n# NAME\n'
-readme = re.sub(r'(?s)# INSTALLATION.*?(?=# DESCRIPTION)', '', readme)
+PREFIX = '''%YOUTUBE-DL(1)
+
+# NAME
+
+youtube\-dl \- download videos from youtube.com or other video platforms
+
+# SYNOPSIS
+
+**youtube-dl** \[OPTIONS\] URL [URL...]
+
+'''
+readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
+readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
 readme = PREFIX + readme
 
 if sys.version_info < (3, 0):

From ec05fee43acc49f4aded253e7072683b268a7965 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 16:38:26 +0100
Subject: [PATCH 242/974] [brightcove] Add shorter URL scheme for other
 extractors

---
 youtube_dl/extractor/aljazeera.py  | 9 +++++----
 youtube_dl/extractor/brightcove.py | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index ace5ec19d..380d429d6 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -24,9 +24,10 @@ class AlJazeeraIE(InfoExtractor):
 
         return {
             '_type': 'url',
-            'url':
-                'http://c.brightcove.com/services/viewer/federated_f9?'
-                '&playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
-                '&%40videoPlayer={0}'.format(brightcove_id),
+            'url': (
+                'brightcove:'
+                'playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
+                '&%40videoPlayer={0}'.format(brightcove_id)
+            ),
             'ie_key': 'Brightcove',
         }
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 1eca00470..003152c4e 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -25,7 +25,7 @@ from ..utils import (
 
 
 class BrightcoveIE(InfoExtractor):
-    _VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*?\?(?P<query>.*)'
+    _VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
     _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
 
     _TESTS = [

From 13d27a42cc8db567ddaaa233c90c3ae68e7f8f0b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 16:45:28 +0100
Subject: [PATCH 243/974] [orf:tvthek] Add support for topic URLs (Fixes #4474)

---
 youtube_dl/extractor/orf.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index a6e722bf5..4fed83bd6 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -17,9 +17,9 @@ from ..utils import (
 class ORFTVthekIE(InfoExtractor):
     IE_NAME = 'orf:tvthek'
     IE_DESC = 'ORF TVthek'
-    _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics/.+?|program/[^/]+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics?/.+?|program/[^/]+)/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
         'playlist': [{
             'md5': '2942210346ed779588f428a92db88712',
@@ -32,8 +32,21 @@ class ORFTVthekIE(InfoExtractor):
                 'upload_date': '20141208',
             },
         }],
-        'skip': 'Blocked outside of Austria',
-    }
+        'skip': 'Blocked outside of Austria / Germany',
+    }, {
+        'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
+        'playlist': [{
+            'md5': '68f543909aea49d621dfc7703a11cfaf',
+            'info_dict': {
+                'id': '7982259',
+                'ext': 'mp4',
+                'title': 'Best of Ingrid Thurnher',
+                'upload_date': '20140527',
+                'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
+            }
+        }],
+        '_skip': 'Blocked outside of Austria / Germany',
+    }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -45,7 +58,9 @@ class ORFTVthekIE(InfoExtractor):
 
         def get_segments(all_data):
             for data in all_data:
-                if data['name'] == 'Tracker::EPISODE_DETAIL_PAGE_OVER_PROGRAM':
+                if data['name'] in (
+                        'Tracker::EPISODE_DETAIL_PAGE_OVER_PROGRAM',
+                        'Tracker::EPISODE_DETAIL_PAGE_OVER_TOPIC'):
                     return data['values']['segments']
 
         sdata = get_segments(all_data)

From 964b190350f217c2e3f157130072ee667b299b24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 16:45:35 +0100
Subject: [PATCH 244/974] release 2014.12.16.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dacc47a78..768dc7599 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.16.1'
+__version__ = '2014.12.16.2'

From 2cb5b03e53eefca4610a86e7c07bd2e0564b6dda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Dec 2014 20:33:23 +0100
Subject: [PATCH 245/974] [test/test_unicode_literals] Remove duplicated
 imports

---
 test/test_unicode_literals.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index 19813e034..7f816698e 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -7,9 +7,7 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import io
-import os
 import re
-import unittest
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 

From ee6dfe8308c967573a9d12d8dbbc1533cec45f51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Dec 2014 19:38:56 +0100
Subject: [PATCH 246/974] Use flake8 instead of pyflakes and pep8

It combines both tools
---
 Makefile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 6fb820c2a..524c5fcda 100644
--- a/Makefile
+++ b/Makefile
@@ -36,12 +36,11 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtu
 	install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish
 
 codetest:
-	PYFLAKES_OUT=$$(pyflakes youtube_dl | grep -v youtube_dl/extractor/__init__.py); \
-	if test -n "$$PYFLAKES_OUT"; then \
-		echo "$$PYFLAKES_OUT"; \
+	FLAKE8_OUT=$$(flake8 --ignore E501 . | grep -v youtube_dl/extractor/__init__.py); \
+	if test -n "$$FLAKE8_OUT"; then \
+		echo "$$FLAKE8_OUT"; \
 		exit 1; \
 	fi
-	pep8 . --ignore E501
 
 test:
 	#nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test

From dc9a441bfa8b2767748a8e3d138ae765e0d7b0cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Dec 2014 19:54:22 +0100
Subject: [PATCH 247/974] Move flake8 configuration to setup.cfg

It will be used when calling flake8 from any directory in the project
---
 Makefile  | 6 +-----
 setup.cfg | 4 ++++
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 524c5fcda..71c8553d8 100644
--- a/Makefile
+++ b/Makefile
@@ -36,11 +36,7 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtu
 	install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish
 
 codetest:
-	FLAKE8_OUT=$$(flake8 --ignore E501 . | grep -v youtube_dl/extractor/__init__.py); \
-	if test -n "$$FLAKE8_OUT"; then \
-		echo "$$FLAKE8_OUT"; \
-		exit 1; \
-	fi
+	flake8 .
 
 test:
 	#nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test
diff --git a/setup.cfg b/setup.cfg
index e57d130e3..8659b5069 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,2 +1,6 @@
 [wheel]
 universal = True
+
+[flake8]
+exclude = youtube_dl/extractor/__init__.py
+ignore = E501

From a16af518735cb0301731900914c84f26823e9372 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Dec 2014 20:38:59 +0100
Subject: [PATCH 248/974] flake8: Add more ignored files

* setup.py: the '__version__' variable is not defined in the script, it is loadded from youtube_dl/version.py
* devscripts/buildserver.py: Produces a lot of messages
---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 8659b5069..898defb9d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,5 +2,5 @@
 universal = True
 
 [flake8]
-exclude = youtube_dl/extractor/__init__.py
+exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py
 ignore = E501

From 876bef5937526c6132fc22fab461d41bbf79014b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 00:04:24 +0100
Subject: [PATCH 249/974] [mit] Modernize

---
 youtube_dl/extractor/mit.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index d354702d1..78787e8f1 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class TechTVMITIE(InfoExtractor):
     IE_NAME = 'techtv.mit.edu'
-    _VALID_URL = r'https?://techtv\.mit\.edu/(videos|embeds)/(?P<id>\d+)'
+    _VALID_URL = r'https?://techtv\.mit\.edu/(?:videos|embeds)/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://techtv.mit.edu/videos/25418-mit-dna-learning-center-set',
@@ -31,8 +31,7 @@ class TechTVMITIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         raw_page = self._download_webpage(
             'http://techtv.mit.edu/videos/%s' % video_id, video_id)
         clean_page = re.compile(r'<!--.*?-->', re.S).sub('', raw_page)
@@ -106,7 +105,7 @@ class OCWMITIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Lecture 7: Multiple Discrete Random Variables: Expectations, Conditioning, Independence',
                 'description': 'In this lecture, the professor discussed multiple random variables, expectations, and binomial distribution.',
-                #'subtitles': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/MIT6_041F11_lec07_300k.mp4.srt'
+                # 'subtitles': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/MIT6_041F11_lec07_300k.mp4.srt'
             }
         },
         {
@@ -116,7 +115,7 @@ class OCWMITIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Session 1: Introduction to Derivatives',
                 'description': 'This section contains lecture video excerpts, lecture notes, an interactive mathlet with supporting documents, and problem solving videos.',
-                #'subtitles': 'http://ocw.mit.edu//courses/mathematics/18-01sc-single-variable-calculus-fall-2010/ocw-18.01-f07-lec01_300k.SRT'
+                # 'subtitles': 'http://ocw.mit.edu//courses/mathematics/18-01sc-single-variable-calculus-fall-2010/ocw-18.01-f07-lec01_300k.SRT'
             }
         }
     ]

From 7af808a5ef4cc17ca30c8c4636660589e0e915ec Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 00:06:41 +0100
Subject: [PATCH 250/974] Improve code style

---
 youtube_dl/downloader/f4m.py        | 2 +-
 youtube_dl/downloader/rtmp.py       | 2 +-
 youtube_dl/extractor/crunchyroll.py | 1 -
 youtube_dl/extractor/cspan.py       | 1 -
 youtube_dl/extractor/tenplay.py     | 1 -
 youtube_dl/extractor/youtube.py     | 4 ++--
 youtube_dl/utils.py                 | 2 +-
 7 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index ef3e0d5f4..f9f6f3e73 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -203,7 +203,7 @@ def write_flv_header(stream, metadata):
     stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
     stream.write(metadata)
     # Magic numbers extracted from the output files produced by AdobeHDS.php
-    #(https://github.com/K-S-V/Scripts)
+    # (https://github.com/K-S-V/Scripts)
     stream.write(b'\x00\x00\x01\x73')
 
 
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 575912675..5346cb9a0 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -185,7 +185,7 @@ class RtmpFD(FileDownloader):
             cursize = os.path.getsize(encodeFilename(tmpfilename))
             if prevsize == cursize and retval == RD_FAILED:
                 break
-             # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
+            # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
             if prevsize == cursize and retval == RD_INCOMPLETE and cursize > 1024:
                 self.to_screen('[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
                 retval = RD_SUCCESS
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 8f1ea02e7..2c7756eb6 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -32,7 +32,6 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _TEST = {
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
-        #'md5': 'b1639fd6ddfaa43788c85f6d1dddd412',
         'info_dict': {
             'id': '645513',
             'ext': 'flv',
diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 541106684..955119d40 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -27,7 +27,6 @@ class CSpanIE(InfoExtractor):
         'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
         # For whatever reason, the served video alternates between
         # two different ones
-        #'md5': 'dbb0f047376d457f2ab8b3929cbb2d0c',
         'info_dict': {
             'id': '340723',
             'ext': 'mp4',
diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py
index 81ba169fb..466155ef8 100644
--- a/youtube_dl/extractor/tenplay.py
+++ b/youtube_dl/extractor/tenplay.py
@@ -8,7 +8,6 @@ class TenPlayIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ten(play)?\.com\.au/.+'
     _TEST = {
         'url': 'http://tenplay.com.au/ten-insider/extra/season-2013/tenplay-tv-your-way',
-        #'md5': 'd68703d9f73dc8fccf3320ab34202590',
         'info_dict': {
             'id': '2695695426001',
             'ext': 'flv',
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 100ecbc0a..550e18733 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -527,8 +527,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 return 's[%s%s%s]' % (starts, ends, steps)
 
             step = None
-            start = '(Never used)'  # Quelch pyflakes warnings - start will be
-                                    # set as soon as step is set
+            # Quelch pyflakes warnings - start will be set when step is set
+            start = '(Never used)'
             for i, prev in zip(idxs[1:], idxs[:-1]):
                 if step is not None:
                     if i - prev == step:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f9938616d..43b7c94ba 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -363,7 +363,7 @@ def encodeArgument(s):
     if not isinstance(s, compat_str):
         # Legacy code that uses byte strings
         # Uncomment the following line after fixing all post processors
-        #assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
+        # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
         s = s.decode('ascii')
     return encodeFilename(s, True)
 

From 37381350f8f6a785df0aae75bef09d628fdb1b1a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 00:08:04 +0100
Subject: [PATCH 251/974] [aljazeera] Add unicode_literals marker

---
 youtube_dl/extractor/aljazeera.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index 380d429d6..612708e25 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from .common import InfoExtractor
 
 

From b1ccbed3d43b10369ba779d8873ed6b85767f3c1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 00:26:04 +0100
Subject: [PATCH 252/974] [nhl] Allow upper-case video IDs (Fixes #4494)

---
 youtube_dl/extractor/nhl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index b2f40344f..d3a4fc513 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -54,7 +54,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
 class NHLIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
 
     _TESTS = [{
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',

From 6e1b9395c69910cefdb4fb080855292cf4b6964a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 10:53:12 +0100
Subject: [PATCH 253/974] [screencastomatic] Add new extractor (Fixes #4497)

---
 youtube_dl/extractor/__init__.py         |  1 +
 youtube_dl/extractor/screencastomatic.py | 50 ++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/screencastomatic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b091505db..c30cbc926 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -342,6 +342,7 @@ from .savefrom import SaveFromIE
 from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
+from .screencastomatic import ScreencastOMaticIE
 from .screenwavemedia import CinemassacreIE, ScreenwaveMediaIE, TeamFourIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
diff --git a/youtube_dl/extractor/screencastomatic.py b/youtube_dl/extractor/screencastomatic.py
new file mode 100644
index 000000000..1dafc4c31
--- /dev/null
+++ b/youtube_dl/extractor/screencastomatic.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    ExtractorError,
+    js_to_json,
+)
+
+
+class ScreencastOMaticIE(InfoExtractor):
+    _VALID_URL = r'https?://screencast-o-matic\.com/watch/(?P<id>[0-9a-zA-Z]+)'
+    _TEST = {
+        'url': 'http://screencast-o-matic.com/watch/c2lD3BeOPl',
+        'md5': '483583cb80d92588f15ccbedd90f0c18',
+        'info_dict': {
+            'id': 'c2lD3BeOPl',
+            'ext': 'mp4',
+            'title': 'Welcome to 3-4 Philosophy @ DECV!',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'as the title says! also: some general info re 1) VCE philosophy and 2) distance learning.',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        setup_js = self._search_regex(
+            r"(?s)jwplayer\('mp4Player'\).setup\((\{.*?\})\);",
+            webpage, 'setup code')
+        data = self._parse_json(setup_js, video_id, transform_source=js_to_json)
+        modes = data['modes']
+        try:
+            video_data = next(
+                m for m in data['modes'] if m.get('type') == 'html5')
+        except StopIteration:
+            raise ExtractorError('Could not find any video entries!')
+        video_url = compat_urlparse.urljoin(url, video_data['config']['file'])
+        thumbnail = data.get('image')
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'url': video_url,
+            'ext': 'mp4',
+            'thumbnail': thumbnail,
+        }

From d6a8160902c2f316b3772648d6eec6bfcb095471 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 10:53:17 +0100
Subject: [PATCH 254/974] release 2014.12.17

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 768dc7599..a87e2dd1d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.16.2'
+__version__ = '2014.12.17'

From 27d67ea2ba3a61718e61787117832f1c1eb60b87 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 11:29:35 +0100
Subject: [PATCH 255/974] [comedycentral] Match URLs with a second ID (fixes
 #4499)

---
 youtube_dl/extractor/comedycentral.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 48e2410b6..8d27af5e5 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -50,7 +50,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
                           )|
                           (?P<interview>
                               extended-interviews/(?P<interID>[0-9a-z]+)/(?:playlist_tds_extended_)?(?P<interview_title>.*?)(/.*?)?)))
-                     (?:[?#].*|$)'''
+                     '''
     _TESTS = [{
         'url': 'http://thedailyshow.cc.com/watch/thu-december-13-2012/kristen-stewart',
         'md5': '4e2f5cb088a83cd8cdb7756132f9739d',
@@ -83,6 +83,9 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://thedailyshow.cc.com/video-playlists/npde3s/the-daily-show-19088-highlights',
         'only_matching': True,
+    }, {
+        'url': 'http://thedailyshow.cc.com/video-playlists/t6d9sg/the-daily-show-20038-highlights/be3cwo',
+        'only_matching': True,
     }, {
         'url': 'http://thedailyshow.cc.com/special-editions/2l8fdb/special-edition---a-look-back-at-food',
         'only_matching': True,

From a91c9b15e3b2b2b3f2f9385b18ae7d75b802e72f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 11:29:52 +0100
Subject: [PATCH 256/974] release 2014.12.17.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a87e2dd1d..6f04cc2d0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.17'
+__version__ = '2014.12.17.1'

From fdc80008100ebcca2a1456d6e8a7064e65f6e69f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 11:38:59 +0100
Subject: [PATCH 257/974] [downloader] Handle a file ./- (Fixes #4498)

---
 youtube_dl/downloader/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index d3e0b0110..584bde732 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -285,7 +285,7 @@ class FileDownloader(object):
         Return True on success and False otherwise
         """
         # Check file already present
-        if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
+        if filename != '-' and self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,

From 7642c08763cda0cadeb812393ed3d1b2c8ecb99a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 11:39:25 +0100
Subject: [PATCH 258/974] release 2014.12.17.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6f04cc2d0..1420af746 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.17.1'
+__version__ = '2014.12.17.2'

From c33bcf2051cac1580d5161a188acc3efd54734ea Mon Sep 17 00:00:00 2001
From: AndroKev <AndroKev@users.noreply.github.com>
Date: Wed, 17 Dec 2014 13:05:19 +0100
Subject: [PATCH 259/974] only add video-id to archive, when successful

Example:
no space left--> youtube-dl adds the id to archive, but the video isn't fully downloaded
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8a6bcd51c..82acfe282 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1145,12 +1145,12 @@ class YoutubeDL(object):
 
             if success:
                 try:
+                    self.record_download_archive(info_dict)
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
                     self.report_error('postprocessing: %s' % str(err))
                     return
 
-        self.record_download_archive(info_dict)
 
     def download(self, url_list):
         """Download a given list of URLs."""

From cd58dc3e56b2efa955bda6ced7105df2bb790e63 Mon Sep 17 00:00:00 2001
From: AndroKev <AndroKev@users.noreply.github.com>
Date: Wed, 17 Dec 2014 13:21:22 +0100
Subject: [PATCH 260/974] Update YoutubeDL.py

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 82acfe282..30f1cd1ee 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1145,11 +1145,11 @@ class YoutubeDL(object):
 
             if success:
                 try:
-                    self.record_download_archive(info_dict)
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
                     self.report_error('postprocessing: %s' % str(err))
                     return
+                self.record_download_archive(info_dict)
 
 
     def download(self, url_list):

From 0e4cb4f406337d3b8879cabb556bde1ec6742b74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 17 Dec 2014 14:55:27 +0100
Subject: [PATCH 261/974] YoutubeDL: style fix

---
 youtube_dl/YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 30f1cd1ee..14ee30e55 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1151,7 +1151,6 @@ class YoutubeDL(object):
                     return
                 self.record_download_archive(info_dict)
 
-
     def download(self, url_list):
         """Download a given list of URLs."""
         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)

From e54a3a2f01d2dec2c1d65d3064dd35e7e62cbf40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 17 Dec 2014 14:56:30 +0100
Subject: [PATCH 262/974] [screencastomatic] Remove unused variable

---
 youtube_dl/extractor/screencastomatic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/screencastomatic.py b/youtube_dl/extractor/screencastomatic.py
index 1dafc4c31..05337421c 100644
--- a/youtube_dl/extractor/screencastomatic.py
+++ b/youtube_dl/extractor/screencastomatic.py
@@ -31,7 +31,6 @@ class ScreencastOMaticIE(InfoExtractor):
             r"(?s)jwplayer\('mp4Player'\).setup\((\{.*?\})\);",
             webpage, 'setup code')
         data = self._parse_json(setup_js, video_id, transform_source=js_to_json)
-        modes = data['modes']
         try:
             video_data = next(
                 m for m in data['modes'] if m.get('type') == 'html5')

From 748ec6672533d80881475186ed292caea9f8e8c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Dec 2014 20:20:40 +0600
Subject: [PATCH 263/974] [theplatform] Extract captions (Closes #4495)

---
 youtube_dl/extractor/theplatform.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index af6ef0033..110ed976d 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 import json
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_str,
 )
@@ -16,7 +16,7 @@ from ..utils import (
 _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'})
 
 
-class ThePlatformIE(InfoExtractor):
+class ThePlatformIE(SubtitlesInfoExtractor):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/[^/]+/
            (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
@@ -66,6 +66,20 @@ class ThePlatformIE(InfoExtractor):
         info_json = self._download_webpage(info_url, video_id)
         info = json.loads(info_json)
 
+        subtitles = {}
+        captions = info.get('captions')
+        if isinstance(captions, list):
+            for caption in captions:
+                lang, src = caption.get('lang'), caption.get('src')
+                if lang and src:
+                    subtitles[lang] = src
+
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
+        subtitles = self.extract_subtitles(video_id, subtitles)
+
         head = meta.find(_x('smil:head'))
         body = meta.find(_x('smil:body'))
 
@@ -117,6 +131,7 @@ class ThePlatformIE(InfoExtractor):
         return {
             'id': video_id,
             'title': info['title'],
+            'subtitles': subtitles,
             'formats': formats,
             'description': info['description'],
             'thumbnail': info['defaultThumbnailUrl'],

From 60e5428925f7299182b4b327b3e88908e33eca1c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 15:36:16 +0100
Subject: [PATCH 264/974] [flake8] Ignore build/ directory

That directory is temporarily generated when building for PyPi and may be present if something goes wrong with the upload.
---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 898defb9d..13dcd8af6 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,5 +2,5 @@
 universal = True
 
 [flake8]
-exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py
+exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build
 ignore = E501

From 5f627b4448f3276089978177a550624bbd5e945f Mon Sep 17 00:00:00 2001
From: Petr Kutalek <petr.kutalek@forers.com>
Date: Wed, 17 Dec 2014 15:52:54 +0100
Subject: [PATCH 265/974] [dvtv] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dvtv.py     | 63 ++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/dvtv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c30cbc926..79f6f6f13 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -88,6 +88,7 @@ from .dotsub import DotsubIE
 from .dreisat import DreiSatIE
 from .drtuber import DrTuberIE
 from .drtv import DRTVIE
+from .dvtv import DVTVIE
 from .dump import DumpIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
new file mode 100644
index 000000000..a52bb3469
--- /dev/null
+++ b/youtube_dl/extractor/dvtv.py
@@ -0,0 +1,63 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import re
+import json
+from .common import InfoExtractor
+from ..utils import (
+	ExtractorError,
+	js_to_json,
+	unescapeHTML
+)
+
+
+class DVTVIE(InfoExtractor):
+	IE_NAME = 'dvtv'
+	IE_DESC = 'http://video.aktualne.cz/dvtv/'
+
+	_VALID_URL = r'http://video\.aktualne\.cz/dvtv/(?P<id>[a-z0-9-]+/r~[0-9a-f]{32})/?'
+
+	_TESTS = [{
+		'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
+		'md5': '75800f964fa0f82939a2914563301f72',
+		'info_dict': {
+			'id': 'e5efe9ca855511e4833a0025900fea04',
+			'ext': 'webm',
+			'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně'
+		}
+	}, {
+		'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
+		'md5': 'd50455195a67a94c57f931360cc68a1b',
+		'info_dict': {
+			'id': '82ed4322849211e4a10c0025900fea04',
+			'ext': 'webm',
+			'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala'
+		}
+	}]
+
+	def _real_extract(self, url):
+		video_id = self._match_id(url)
+		webpage = self._download_webpage(url, video_id)
+
+		code = self._search_regex(r'embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});', webpage, 'video JSON', flags=re.DOTALL)
+		payload = self._parse_json(code, video_id, transform_source=js_to_json)
+		formats = []
+		for source in payload['sources']:
+			formats.append({
+				'url': source['file'],
+				'ext': source['type'][6:],
+				'format': '%s %s' % (source['type'][6:], source['label']),
+				'format_id': '%s-%s' % (source['type'][6:], source['label']),
+				'resolution': source['label'],
+				'fps': 25,
+				'preference': -1 if source['type'][6:] == 'mp4' and source['label'] == '720p' else -2
+			})
+
+		return {
+			'id': video_id[-32:],
+			'display_id': video_id[:-35],
+			'title': unescapeHTML(payload['title']),
+			'thumbnail': 'http:%s' % payload['image'],
+			'formats': formats
+		}

From b9465395cbc222fc9609b7881098c7246ee93989 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 23:18:06 +0100
Subject: [PATCH 266/974] [dvtv] PEP8 and correct format sorting (#4502)

---
 youtube_dl/extractor/dvtv.py | 96 ++++++++++++++++++------------------
 1 file changed, 48 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index a52bb3469..af552831c 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -2,62 +2,62 @@
 
 from __future__ import unicode_literals
 
-import re
-import json
 from .common import InfoExtractor
 from ..utils import (
-	ExtractorError,
-	js_to_json,
-	unescapeHTML
+    js_to_json,
+    unescapeHTML
 )
 
 
 class DVTVIE(InfoExtractor):
-	IE_NAME = 'dvtv'
-	IE_DESC = 'http://video.aktualne.cz/dvtv/'
+    IE_NAME = 'dvtv'
+    IE_DESC = 'http://video.aktualne.cz/dvtv/'
 
-	_VALID_URL = r'http://video\.aktualne\.cz/dvtv/(?P<id>[a-z0-9-]+/r~[0-9a-f]{32})/?'
+    _VALID_URL = r'http://video\.aktualne\.cz/dvtv/(?P<id>[a-z0-9-]+/r~[0-9a-f]{32})/?'
 
-	_TESTS = [{
-		'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
-		'md5': '75800f964fa0f82939a2914563301f72',
-		'info_dict': {
-			'id': 'e5efe9ca855511e4833a0025900fea04',
-			'ext': 'webm',
-			'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně'
-		}
-	}, {
-		'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
-		'md5': 'd50455195a67a94c57f931360cc68a1b',
-		'info_dict': {
-			'id': '82ed4322849211e4a10c0025900fea04',
-			'ext': 'webm',
-			'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala'
-		}
-	}]
+    _TESTS = [{
+        'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
+        'md5': '75800f964fa0f82939a2914563301f72',
+        'info_dict': {
+            'id': 'e5efe9ca855511e4833a0025900fea04',
+            'ext': 'webm',
+            'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně'
+        }
+    }, {
+        'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
+        'md5': '6388f1941b48537dbd28791f712af8bf',
+        'info_dict': {
+            'id': '82ed4322849211e4a10c0025900fea04',
+            'ext': 'mp4',
+            'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala'
+        }
+    }]
 
-	def _real_extract(self, url):
-		video_id = self._match_id(url)
-		webpage = self._download_webpage(url, video_id)
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
 
-		code = self._search_regex(r'embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});', webpage, 'video JSON', flags=re.DOTALL)
-		payload = self._parse_json(code, video_id, transform_source=js_to_json)
-		formats = []
-		for source in payload['sources']:
-			formats.append({
-				'url': source['file'],
-				'ext': source['type'][6:],
-				'format': '%s %s' % (source['type'][6:], source['label']),
-				'format_id': '%s-%s' % (source['type'][6:], source['label']),
-				'resolution': source['label'],
-				'fps': 25,
-				'preference': -1 if source['type'][6:] == 'mp4' and source['label'] == '720p' else -2
-			})
+        code = self._search_regex(
+            r'(?s)embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});',
+            webpage, 'video JSON')
+        payload = self._parse_json(code, video_id, transform_source=js_to_json)
+        formats = []
+        for source in payload['sources']:
+            ext = source['type'][6:]
+            formats.append({
+                'url': source['file'],
+                'ext': ext,
+                'format': '%s %s' % (ext, source['label']),
+                'format_id': '%s-%s' % (ext, source['label']),
+                'height': int(source['label'].rstrip('p')),
+                'fps': 25,
+            })
+        self._sort_formats(formats)
 
-		return {
-			'id': video_id[-32:],
-			'display_id': video_id[:-35],
-			'title': unescapeHTML(payload['title']),
-			'thumbnail': 'http:%s' % payload['image'],
-			'formats': formats
-		}
+        return {
+            'id': video_id[-32:],
+            'display_id': video_id[:-35],
+            'title': unescapeHTML(payload['title']),
+            'thumbnail': 'http:%s' % payload['image'],
+            'formats': formats
+        }

From 87c4c21e7583b8d6af04dda822768c226df4ca25 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 23:37:57 +0100
Subject: [PATCH 267/974] Credit @petrkutalek for dvtv (#4502)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 5d45e02fc..a4fa76801 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -93,3 +93,4 @@ Zack Fernandes
 cryptonaut
 Adrian Kretz
 Mathias Rav
+Petr Kutalek

From a22524b00438fc7dcde1e041d7a576eb443b4743 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Dec 2014 21:25:42 +0600
Subject: [PATCH 268/974] [twitch] Add support for vods (Closes #4512)

---
 youtube_dl/extractor/twitch.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 715f9930c..b11a1d561 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -22,6 +22,7 @@ class TwitchIE(InfoExtractor):
     _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?twitch\.tv/
         (?:
             (?P<channelid>[^/]+)|
+            (?:(?:[^/]+)/v/(?P<vodid>[^/]+))|
             (?:(?:[^/]+)/b/(?P<videoid>[^/]+))|
             (?:(?:[^/]+)/c/(?P<chapterid>[^/]+))
         )
@@ -70,11 +71,24 @@ class TwitchIE(InfoExtractor):
     def _extract_media(self, item, item_id):
         ITEMS = {
             'a': 'video',
+            'v': 'vod',
             'c': 'chapter',
         }
         info = self._extract_info(self._download_json(
             '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
             'Downloading %s info JSON' % ITEMS[item]))
+
+        if item == 'v':
+            access_token = self._download_json(
+                '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
+                'Downloading %s access token' % ITEMS[item])
+            formats = self._extract_m3u8_formats(
+                'http://usher.twitch.tv/vod/%s?nauth=%s&nauthsig=%s'
+                % (item_id, access_token['token'], access_token['sig']),
+                item_id, 'mp4')
+            info['formats'] = formats
+            return info
+
         response = self._download_json(
             '%s/api/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
             'Downloading %s playlist JSON' % ITEMS[item])
@@ -209,6 +223,8 @@ class TwitchIE(InfoExtractor):
             """
         elif mobj.group('videoid'):
             return self._extract_media('a', mobj.group('videoid'))
+        elif mobj.group('vodid'):
+            return self._extract_media('v', mobj.group('vodid'))
         elif mobj.group('channelid'):
             channel_id = mobj.group('channelid')
             info = self._download_json(

From c58843b3a15393a46e74c8c1301b828dfcb5002c Mon Sep 17 00:00:00 2001
From: Will Glynn <will@willglynn.com>
Date: Thu, 18 Dec 2014 22:28:13 -0600
Subject: [PATCH 269/974] [nerdcubed] Add new extractor

nerdcubed.co.uk describes videos in a single a feed.json file, providing
references to and metadata on >1300 YouTube videos spread across 3 main
channels as well as guest appareances on other channels via a single HTTP
request.

NerdCubedFeedIE transforms this feed into a youtube-dl playlist, preserving
information present in the upstream JSON (allowing zero-cost title/date
matches) and ultimately referencing the embedded YouTube videos.
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/nerdcubed.py | 36 +++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/nerdcubed.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79f6f6f13..76f13bf52 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -264,6 +264,7 @@ from .nbc import (
 )
 from .ndr import NDRIE
 from .ndtv import NDTVIE
+from .nerdcubed import NerdCubedFeedIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
 from .nfb import NFBIE
diff --git a/youtube_dl/extractor/nerdcubed.py b/youtube_dl/extractor/nerdcubed.py
new file mode 100644
index 000000000..9f2e678e0
--- /dev/null
+++ b/youtube_dl/extractor/nerdcubed.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import datetime
+
+from .common import InfoExtractor
+
+
+class NerdCubedFeedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nerdcubed\.co\.uk/feed\.json'
+    _TEST = {
+        'url': 'http://www.nerdcubed.co.uk/feed.json',
+        'info_dict': {
+            'title': 'nerdcubed.co.uk feed',
+        },
+        'playlist_mincount': 1300,
+    }
+
+    def _real_extract(self, url):
+        feed = self._download_json(url, url, "Downloading NerdCubed JSON feed")
+        
+        entries = [{
+            '_type': 'url',
+            'title': feed_entry['title'],
+            'uploader': feed_entry['source']['name'] if feed_entry['source'] else None,
+            'upload_date': datetime.datetime.strptime(feed_entry['date'], '%Y-%m-%d').strftime('%Y%m%d'),
+            'url': "http://www.youtube.com/watch?v=" + feed_entry['youtube_id'],
+        } for feed_entry in feed]
+
+        return {
+            '_type': 'playlist',
+            'title': 'nerdcubed.co.uk feed',
+            'id': 'nerdcubed-feed',
+            'entries': entries,
+        }
+

From a2a4bae9298c77f5262ae45f6fc0fc8cb243d4af Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 19 Dec 2014 10:32:20 +0100
Subject: [PATCH 270/974] Credit @willglynn for nerdcubed (#4515)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index a4fa76801..6ea958fce 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -94,3 +94,4 @@ cryptonaut
 Adrian Kretz
 Mathias Rav
 Petr Kutalek
+Will Glynn

From 16ea8179681263d570e632d3ce523ed23ddbf204 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Dec 2014 21:56:44 +0600
Subject: [PATCH 271/974] [xtube] Fix and modernize (Closes #4489)

---
 youtube_dl/extractor/xtube.py | 62 ++++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index f9d98b83f..cf74d4fd5 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -1,11 +1,11 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
+    compat_urllib_parse,
 )
 from ..utils import (
     parse_duration,
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class XTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<id>[^/?&]+))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<id>[^/?&#]+))'
     _TEST = {
         'url': 'http://www.xtube.com/watch.php?v=kVTUy_G222_',
         'md5': '092fbdd3cbe292c920ef6fc6a8a9cdab',
@@ -30,41 +30,49 @@ class XTubeIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        url = 'http://www.' + mobj.group('url')
+        video_id = self._match_id(url)
 
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
-        video_title = self._html_search_regex(r'<p class="title">([^<]+)', webpage, 'title')
+        video_title = self._html_search_regex(
+            r'<p class="title">([^<]+)', webpage, 'title')
         video_uploader = self._html_search_regex(
-            r'so_s\.addVariable\("owner_u", "([^"]+)', webpage, 'uploader', fatal=False)
+            [r"var\s+contentOwnerId\s*=\s*'([^']+)",
+             r'By:\s*<a href="/community/profile\.php?user=([^"]+)'],
+            webpage, 'uploader', fatal=False)
         video_description = self._html_search_regex(
-            r'<p class="fieldsDesc">([^<]+)', webpage, 'description', fatal=False)
+            r'<p class="fieldsDesc">([^<]+)',
+            webpage, 'description', fatal=False)
         duration = parse_duration(self._html_search_regex(
-            r'<span class="bold">Runtime:</span> ([^<]+)</p>', webpage, 'duration', fatal=False))
-        view_count = self._html_search_regex(
-            r'<span class="bold">Views:</span> ([\d,\.]+)</p>', webpage, 'view count', fatal=False)
-        if view_count:
-            view_count = str_to_int(view_count)
-        comment_count = self._html_search_regex(
-            r'<div id="commentBar">([\d,\.]+) Comments</div>', webpage, 'comment count', fatal=False)
-        if comment_count:
-            comment_count = str_to_int(comment_count)
+            r'<span class="bold">Runtime:</span> ([^<]+)</p>',
+            webpage, 'duration', fatal=False))
+        view_count = str_to_int(self._html_search_regex(
+            r'<span class="bold">Views:</span> ([\d,\.]+)</p>',
+            webpage, 'view count', fatal=False))
+        comment_count = str_to_int(self._html_search_regex(
+            r'<div id="commentBar">([\d,\.]+) Comments</div>',
+            webpage, 'comment count', fatal=False))
 
-        player_quality_option = json.loads(self._html_search_regex(
-            r'playerQualityOption = ({.+?});', webpage, 'player quality option'))
-
-        QUALITIES = ['3gp', 'mp4_normal', 'mp4_high', 'flv', 'mp4_ultra', 'mp4_720', 'mp4_1080']
-        formats = [
-            {
-                'url': furl,
+        formats = []
+        for format_id, video_url in re.findall(
+                r'flashvars\.quality_(.+?)\s*=\s*"([^"]+)"', webpage):
+            fmt = {
+                'url': compat_urllib_parse.unquote(video_url),
                 'format_id': format_id,
-                'preference': QUALITIES.index(format_id) if format_id in QUALITIES else -1,
-            } for format_id, furl in player_quality_option.items()
-        ]
+            }
+            m = re.search(r'^(?P<height>\d+)[pP]', format_id)
+            if m:
+                fmt['height'] = int(m.group('height'))
+            formats.append(fmt)
+
+        if not formats:
+            video_url = compat_urllib_parse.unquote(self._search_regex(
+                r'flashvars\.video_url\s*=\s*"([^"]+)"',
+                webpage, 'video URL'))
+            formats.append({'url': video_url})
+
         self._sort_formats(formats)
 
         return {

From 1ff30d7b79504035bf8f642668361ca71d905ab7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Dec 2014 18:30:56 +0600
Subject: [PATCH 272/974] [npo] Add support for streams (Closes #4276)

---
 youtube_dl/extractor/npo.py | 88 +++++++++++++++++++++++++++----------
 1 file changed, 65 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index ce31694a5..8da76ae45 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -9,6 +9,7 @@ from ..utils import (
     qualities,
     strip_jsonp,
     url_basename,
+    fix_xml_ampersands,
 )
 
 
@@ -51,7 +52,21 @@ class NPOIE(InfoExtractor):
                 'upload_date': '20130225',
                 'duration': 3000,
             },
-        }
+        },
+        {
+            'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706',
+            'info_dict': {
+                'id': 'WO_VPRO_043706',
+                'ext': 'wmv',
+                'title': 'De nieuwe mens - Deel 1',
+                'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b',
+                'duration': 4680,
+            },
+            'params': {
+                # mplayer mms download
+                'skip_download': True,
+            }
+        },
     ]
 
     def _real_extract(self, url):
@@ -74,31 +89,58 @@ class NPOIE(InfoExtractor):
         token = self._search_regex(r'npoplayer\.token = "(.+?)"', token_page, 'token')
 
         formats = []
-        quality = qualities(['adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std'])
-        for format_id in metadata['pubopties']:
-            format_info = self._download_json(
-                'http://ida.omroep.nl/odi/?prid=%s&puboptions=%s&adaptive=yes&token=%s' % (video_id, format_id, token),
-                video_id, 'Downloading %s JSON' % format_id)
-            if format_info.get('error_code', 0) or format_info.get('errorcode', 0):
-                continue
-            streams = format_info.get('streams')
-            if streams:
-                video_info = self._download_json(
-                    streams[0] + '&type=json',
-                    video_id, 'Downloading %s stream JSON' % format_id)
-            else:
-                video_info = format_info
-            video_url = video_info.get('url')
-            if not video_url:
-                continue
-            if format_id == 'adaptive':
-                formats.extend(self._extract_m3u8_formats(video_url, video_id))
-            else:
+
+        pubopties = metadata.get('pubopties')
+        if pubopties:
+            quality = qualities(['adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std'])
+            for format_id in pubopties:
+                format_info = self._download_json(
+                    'http://ida.omroep.nl/odi/?prid=%s&puboptions=%s&adaptive=yes&token=%s'
+                    % (video_id, format_id, token),
+                    video_id, 'Downloading %s JSON' % format_id)
+                if format_info.get('error_code', 0) or format_info.get('errorcode', 0):
+                    continue
+                streams = format_info.get('streams')
+                if streams:
+                    video_info = self._download_json(
+                        streams[0] + '&type=json',
+                        video_id, 'Downloading %s stream JSON' % format_id)
+                else:
+                    video_info = format_info
+                video_url = video_info.get('url')
+                if not video_url:
+                    continue
+                if format_id == 'adaptive':
+                    formats.extend(self._extract_m3u8_formats(video_url, video_id))
+                else:
+                    formats.append({
+                        'url': video_url,
+                        'format_id': format_id,
+                        'quality': quality(format_id),
+                    })
+
+        streams = metadata.get('streams')
+        if streams:
+            for i, stream in enumerate(streams):
+                stream_url = stream.get('url')
+                if not stream_url:
+                    continue
+                asx = self._download_xml(
+                    stream_url, video_id,
+                    'Downloading stream %d ASX playlist' % i,
+                    transform_source=fix_xml_ampersands)
+                ref = asx.find('./ENTRY/Ref')
+                if ref is None:
+                    continue
+                video_url = ref.get('href')
+                if not video_url:
+                    continue
                 formats.append({
                     'url': video_url,
-                    'format_id': format_id,
-                    'quality': quality(format_id),
+                    'ext': stream.get('formaat', 'asf'),
+                    'quality': stream.get('kwaliteit'),
                 })
+
         self._sort_formats(formats)
 
         return {

From 1d4247f64e4124476f807e9eeb58fd401ffcd8ae Mon Sep 17 00:00:00 2001
From: Petr Kutalek <petr.kutalek@forers.com>
Date: Sun, 21 Dec 2014 01:24:05 +0100
Subject: [PATCH 273/974] [dvtv] Add support for playlists

---
 youtube_dl/extractor/dvtv.py | 118 ++++++++++++++++++++++++++---------
 1 file changed, 89 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index af552831c..2449b03b5 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -2,62 +2,122 @@
 
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     js_to_json,
-    unescapeHTML
+    unescapeHTML,
+    ExtractorError
 )
 
 
 class DVTVIE(InfoExtractor):
     IE_NAME = 'dvtv'
-    IE_DESC = 'http://video.aktualne.cz/dvtv/'
+    IE_DESC = 'http://video.aktualne.cz/'
 
-    _VALID_URL = r'http://video\.aktualne\.cz/dvtv/(?P<id>[a-z0-9-]+/r~[0-9a-f]{32})/?'
+    _VALID_URL = r'http://video\.aktualne\.cz/.*/r~(?P<id>[0-9a-f]{32})/'
 
     _TESTS = [{
         'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
-        'md5': '75800f964fa0f82939a2914563301f72',
+        'md5': '67cb83e4a955d36e1b5d31993134a0c2',
         'info_dict': {
-            'id': 'e5efe9ca855511e4833a0025900fea04',
-            'ext': 'webm',
+            'id': 'dc0768de855511e49e4b0025900fea04',
+            'ext': 'mp4',
             'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně'
         }
     }, {
         'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
         'md5': '6388f1941b48537dbd28791f712af8bf',
         'info_dict': {
-            'id': '82ed4322849211e4a10c0025900fea04',
+            'id': '72c02230849211e49f60002590604f2e',
             'ext': 'mp4',
             'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala'
         }
+    }, {
+        'url': 'http://video.aktualne.cz/dvtv/dvtv-16-12-2014-utok-talibanu-boj-o-kliniku-uprchlici/r~973eb3bc854e11e498be002590604f2e/',
+        'info_dict': {
+            'title': 'DVTV 16. 12. 2014: útok Talibanu, boj o kliniku, uprchlíci',
+            'id': '973eb3bc854e11e498be002590604f2e'
+        },
+        'playlist': [{
+            'md5': 'da7ca6be4935532241fa9520b3ad91e4',
+            'info_dict': {
+                'id': 'b0b40906854d11e4bdad0025900fea04',
+                'ext': 'mp4',
+                'title': 'Drtinová Veselovský TV 16. 12. 2014: Témata dne'
+            }
+        }, {
+            'md5': '5f7652a08b05009c1292317b449ffea2',
+            'info_dict': {
+                'id': '420ad9ec854a11e4bdad0025900fea04',
+                'ext': 'mp4',
+                'title': 'Školní masakr možná změní boj s Talibanem, říká novinářka'
+            }
+        }, {
+            'md5': '498eb9dfa97169f409126c617e2a3d64',
+            'info_dict': {
+                'id': '95d35580846a11e4b6d20025900fea04',
+                'ext': 'mp4',
+                'title': 'Boj o kliniku: Veřejný zájem, nebo právo na majetek?'
+            }
+        }, {
+            'md5': 'b8dc6b744844032dab6ba3781a7274b9',
+            'info_dict': {
+                'id': '6fe14d66853511e4833a0025900fea04',
+                'ext': 'mp4',
+                'title': 'Pánek: Odmítání syrských uprchlíků je ostudou české vlády'
+            }
+        }]
     }]
 
+    def _parse_video_metadata(self, js, video_id):
+        metadata = self._parse_json(js, video_id, transform_source=js_to_json)
+
+        formats = []
+        for video in metadata['sources']:
+            ext = video['type'][6:]
+            formats.append({
+                'url': video['file'],
+                'ext': ext,
+                'format': '%s %s' % (ext, video['label']),
+                'format_id': '%s-%s' % (ext, video['label']),
+                'height': int(video['label'].rstrip('p')),
+                'fps': 25,
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': metadata['mediaid'],
+            'title': unescapeHTML(metadata['title']),
+            'thumbnail': 'http:%s' % metadata['image'],
+            'formats': formats
+        }
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        code = self._search_regex(
+        # singe video
+        items =  re.findall(
             r'(?s)embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});',
-            webpage, 'video JSON')
-        payload = self._parse_json(code, video_id, transform_source=js_to_json)
-        formats = []
-        for source in payload['sources']:
-            ext = source['type'][6:]
-            formats.append({
-                'url': source['file'],
-                'ext': ext,
-                'format': '%s %s' % (ext, source['label']),
-                'format_id': '%s-%s' % (ext, source['label']),
-                'height': int(source['label'].rstrip('p')),
-                'fps': 25,
-            })
-        self._sort_formats(formats)
+            webpage)
 
-        return {
-            'id': video_id[-32:],
-            'display_id': video_id[:-35],
-            'title': unescapeHTML(payload['title']),
-            'thumbnail': 'http:%s' % payload['image'],
-            'formats': formats
-        }
+        if items:
+            return self._parse_video_metadata(items[0], video_id)
+
+        # playlist
+        items = re.findall(
+            r'(?s)BBX\.context\.assets\[\'[0-9a-f]{32}\'\]\.push\((\{.+?\})\);',
+            webpage)
+
+        if items:
+            return {
+                '_type': 'playlist',
+                'id': video_id,
+                'title': self._og_search_title(webpage),
+                'entries': [self._parse_video_metadata(i, video_id) for i in items]
+            }
+
+        raise ExtractorError('Could not find neither video nor playlist for requested ID')

From 081d6e47840fb923ed6f565fa56dd5def9b9a184 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Dec 2014 07:33:58 +0600
Subject: [PATCH 274/974] [dvtv] Simplify

---
 youtube_dl/extractor/dvtv.py | 36 +++++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index 2449b03b5..5f8ac3e46 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -1,5 +1,4 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
 import re
@@ -8,7 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     js_to_json,
     unescapeHTML,
-    ExtractorError
+    ExtractorError,
 )
 
 
@@ -16,7 +15,7 @@ class DVTVIE(InfoExtractor):
     IE_NAME = 'dvtv'
     IE_DESC = 'http://video.aktualne.cz/'
 
-    _VALID_URL = r'http://video\.aktualne\.cz/.*/r~(?P<id>[0-9a-f]{32})/'
+    _VALID_URL = r'http://video\.aktualne\.cz/(?:[^/]+/)+r~(?P<id>[0-9a-f]{32})'
 
     _TESTS = [{
         'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
@@ -24,7 +23,7 @@ class DVTVIE(InfoExtractor):
         'info_dict': {
             'id': 'dc0768de855511e49e4b0025900fea04',
             'ext': 'mp4',
-            'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně'
+            'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně',
         }
     }, {
         'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
@@ -32,13 +31,13 @@ class DVTVIE(InfoExtractor):
         'info_dict': {
             'id': '72c02230849211e49f60002590604f2e',
             'ext': 'mp4',
-            'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala'
+            'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala',
         }
     }, {
         'url': 'http://video.aktualne.cz/dvtv/dvtv-16-12-2014-utok-talibanu-boj-o-kliniku-uprchlici/r~973eb3bc854e11e498be002590604f2e/',
         'info_dict': {
             'title': 'DVTV 16. 12. 2014: útok Talibanu, boj o kliniku, uprchlíci',
-            'id': '973eb3bc854e11e498be002590604f2e'
+            'id': '973eb3bc854e11e498be002590604f2e',
         },
         'playlist': [{
             'md5': 'da7ca6be4935532241fa9520b3ad91e4',
@@ -68,7 +67,10 @@ class DVTVIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Pánek: Odmítání syrských uprchlíků je ostudou české vlády'
             }
-        }]
+        }],
+    }, {
+        'url': 'http://video.aktualne.cz/v-cechach-poprve-zazni-zelenkova-zrestaurovana-mse/r~45b4b00483ec11e4883b002590604f2e/',
+        'only_matching': True,
     }]
 
     def _parse_video_metadata(self, js, video_id):
@@ -80,7 +82,6 @@ class DVTVIE(InfoExtractor):
             formats.append({
                 'url': video['file'],
                 'ext': ext,
-                'format': '%s %s' % (ext, video['label']),
                 'format_id': '%s-%s' % (ext, video['label']),
                 'height': int(video['label'].rstrip('p')),
                 'fps': 25,
@@ -91,25 +92,26 @@ class DVTVIE(InfoExtractor):
         return {
             'id': metadata['mediaid'],
             'title': unescapeHTML(metadata['title']),
-            'thumbnail': 'http:%s' % metadata['image'],
+            'thumbnail': self._proto_relative_url(metadata['image']),
             'formats': formats
         }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        # singe video
-        items =  re.findall(
-            r'(?s)embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});',
-            webpage)
+        # single video
+        item = self._search_regex(
+            r"(?s)embedData[0-9a-f]{32}\['asset'\]\s*=\s*(\{.+?\});",
+            webpage, 'video', default=None, fatal=False)
 
-        if items:
-            return self._parse_video_metadata(items[0], video_id)
+        if item:
+            return self._parse_video_metadata(item, video_id)
 
         # playlist
         items = re.findall(
-            r'(?s)BBX\.context\.assets\[\'[0-9a-f]{32}\'\]\.push\((\{.+?\})\);',
+            r"(?s)BBX\.context\.assets\['[0-9a-f]{32}'\]\.push\(({.+?})\);",
             webpage)
 
         if items:
@@ -120,4 +122,4 @@ class DVTVIE(InfoExtractor):
                 'entries': [self._parse_video_metadata(i, video_id) for i in items]
             }
 
-        raise ExtractorError('Could not find neither video nor playlist for requested ID')
+        raise ExtractorError('Could not find neither video nor playlist')

From 2c2a42587b1755624a75a4b748c59a24f8b146aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Dec 2014 07:38:55 +0600
Subject: [PATCH 275/974] [dvtv] Fix thumbnail scheme

---
 youtube_dl/extractor/dvtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index 5f8ac3e46..c1a4bc757 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -92,7 +92,7 @@ class DVTVIE(InfoExtractor):
         return {
             'id': metadata['mediaid'],
             'title': unescapeHTML(metadata['title']),
-            'thumbnail': self._proto_relative_url(metadata['image']),
+            'thumbnail': self._proto_relative_url(metadata['image'], 'http:'),
             'formats': formats
         }
 

From ee45625290c678c53b7ee86d2adba40847b79573 Mon Sep 17 00:00:00 2001
From: MaxReimann <max.reimann@student.hpi.uni-potsdam.de>
Date: Sun, 21 Dec 2014 11:01:28 +0100
Subject: [PATCH 276/974] Add extractor for teletask

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/teletask.py | 59 ++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/teletask.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 76f13bf52..073ac1fab 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -405,6 +405,7 @@ from .ted import TEDIE
 from .telebruxelles import TeleBruxellesIE
 from .telecinco import TelecincoIE
 from .telemb import TeleMBIE
+from .teletask import TeleTaskIE
 from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .tf1 import TF1IE
diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py
new file mode 100644
index 000000000..1c42ac6f4
--- /dev/null
+++ b/youtube_dl/extractor/teletask.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re 
+import datetime
+
+from .common import InfoExtractor
+
+
+class TeleTaskIE(InfoExtractor):
+    _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P<id>[0-9]+)/'
+    _TEST = {
+        'url': 'http://www.tele-task.de/archive/video/html5/26168/', 
+        'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+        'info_dict': {
+            'id': '26168',
+            'ext': 'mp4',
+            'title': 'Duplicate Detection',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'date': '20141218',
+            # TODO more properties, either as:
+            # * A value
+            # * MD5 checksum; start the string with md5:
+            # * A regular expression; start the string with re:
+            # * Any Python type (for example int or float)
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        
+        lecture_url = self._html_search_regex(
+                    r'href="([^"]+)" itemprop="name">', webpage, 'title')
+        lecture_id = re.search("([0-9]+)/",lecture_url).group(1)
+        overview_page = self._download_webpage("http://www.tele-task.de" + lecture_url, 
+            lecture_id)
+        
+        title = self._html_search_regex(
+            r'itemprop="name">([^"]+)</a>', webpage, 'title')
+        url = self._html_search_regex(
+            r'class="speaker".*?src="([^"]+)"', webpage, 'video_url', flags=re.DOTALL)
+        description = self._html_search_regex(
+            r'Description of the series:</p>([^"]+)</div>', overview_page, 
+            'description',flags=re.DOTALL)
+        
+        date = self._html_search_regex(
+            r'<td class="label">Date:</td><td>([^"]+)</td>', webpage, 'date')
+        date = datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y%m%d')
+
+        
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'url': url,
+            'upload_date': date,
+        }
\ No newline at end of file

From 3f7421b71b0d7da917a92b6f37985bc26c5dc11d Mon Sep 17 00:00:00 2001
From: MaxReimann <max.reimann@student.hpi.uni-potsdam.de>
Date: Sun, 21 Dec 2014 11:13:59 +0100
Subject: [PATCH 277/974] fix test and remove lengthy description

---
 youtube_dl/extractor/teletask.py | 20 ++------------------
 1 file changed, 2 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py
index 1c42ac6f4..6920b871e 100644
--- a/youtube_dl/extractor/teletask.py
+++ b/youtube_dl/extractor/teletask.py
@@ -11,18 +11,12 @@ class TeleTaskIE(InfoExtractor):
     _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P<id>[0-9]+)/'
     _TEST = {
         'url': 'http://www.tele-task.de/archive/video/html5/26168/', 
-        'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+        'md5': '290ef69fb2792e481169c3958dbfbd57',
         'info_dict': {
             'id': '26168',
             'ext': 'mp4',
             'title': 'Duplicate Detection',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'date': '20141218',
-            # TODO more properties, either as:
-            # * A value
-            # * MD5 checksum; start the string with md5:
-            # * A regular expression; start the string with re:
-            # * Any Python type (for example int or float)
+            'upload_date': '20141218',
         }
     }
 
@@ -30,19 +24,10 @@ class TeleTaskIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         
-        lecture_url = self._html_search_regex(
-                    r'href="([^"]+)" itemprop="name">', webpage, 'title')
-        lecture_id = re.search("([0-9]+)/",lecture_url).group(1)
-        overview_page = self._download_webpage("http://www.tele-task.de" + lecture_url, 
-            lecture_id)
-        
         title = self._html_search_regex(
             r'itemprop="name">([^"]+)</a>', webpage, 'title')
         url = self._html_search_regex(
             r'class="speaker".*?src="([^"]+)"', webpage, 'video_url', flags=re.DOTALL)
-        description = self._html_search_regex(
-            r'Description of the series:</p>([^"]+)</div>', overview_page, 
-            'description',flags=re.DOTALL)
         
         date = self._html_search_regex(
             r'<td class="label">Date:</td><td>([^"]+)</td>', webpage, 'date')
@@ -53,7 +38,6 @@ class TeleTaskIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'description': description,
             'url': url,
             'upload_date': date,
         }
\ No newline at end of file

From 33b53b60212ca28f589616f1acb650b323a18f11 Mon Sep 17 00:00:00 2001
From: MaxReimann <max.reimann@student.hpi.uni-potsdam.de>
Date: Sun, 21 Dec 2014 12:26:47 +0100
Subject: [PATCH 278/974] [teletask] Add new extractor

---
 youtube_dl/extractor/teletask.py | 62 ++++++++++++++++++++++----------
 1 file changed, 43 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py
index 6920b871e..aa5535042 100644
--- a/youtube_dl/extractor/teletask.py
+++ b/youtube_dl/extractor/teletask.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
-
-import re 
+import re
 import datetime
 
 from .common import InfoExtractor
@@ -10,34 +9,59 @@ from .common import InfoExtractor
 class TeleTaskIE(InfoExtractor):
     _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P<id>[0-9]+)/'
     _TEST = {
-        'url': 'http://www.tele-task.de/archive/video/html5/26168/', 
-        'md5': '290ef69fb2792e481169c3958dbfbd57',
+        'url': 'http://www.tele-task.de/archive/video/html5/26168/',
         'info_dict': {
-            'id': '26168',
-            'ext': 'mp4',
             'title': 'Duplicate Detection',
-            'upload_date': '20141218',
-        }
+        },
+        'playlist': [{
+            'md5': '290ef69fb2792e481169c3958dbfbd57',
+            'info_dict': {
+                'title': 'Duplicate Detection',
+                'upload_date': '20141218',
+                'id': 'speaker_26168',
+                'ext': 'mp4',
+            }
+        },
+            {
+            'md5': 'e1e7218c5f0e4790015a437fcf6c71b4',
+            'info_dict': {
+                'title': 'Duplicate Detection',
+                'upload_date': '20141218',
+                'id': 'slides_26168',
+                'ext': 'mp4',
+            }
+        }]
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        
+        lecture_id = self._match_id(url)
+        webpage = self._download_webpage(url, lecture_id)
+
         title = self._html_search_regex(
             r'itemprop="name">([^"]+)</a>', webpage, 'title')
-        url = self._html_search_regex(
-            r'class="speaker".*?src="([^"]+)"', webpage, 'video_url', flags=re.DOTALL)
-        
+        url_speaker = self._html_search_regex(
+            r'class="speaker".*?src="([^"]+)"', webpage, 'video_url_speaker', flags=re.DOTALL)
+        url_slides = self._html_search_regex(
+            r'class="slides".*?src="([^"]+)"', webpage, 'video_url_slides', flags=re.DOTALL)
         date = self._html_search_regex(
             r'<td class="label">Date:</td><td>([^"]+)</td>', webpage, 'date')
         date = datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y%m%d')
 
-        
+        entries = [{
+            'title': title,
+            'upload_date': date,
+            'id': "speaker_"+lecture_id,
+            'url': url_speaker,
+        },
+            {
+            'title': title,
+            'upload_date': date,
+            'id': "slides_"+lecture_id,
+            'url': url_slides}]
 
         return {
-            'id': video_id,
+            '_type': "playlist",
+            'id': lecture_id,
             'title': title,
-            'url': url,
-            'upload_date': date,
-        }
\ No newline at end of file
+            'entries': entries,
+        }

From 3aa9176f082e2e2909826a61a80a0923b4bd88d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 12:33:30 +0100
Subject: [PATCH 279/974] [yahoo] Improve video id detection (fixes #4521)

---
 youtube_dl/extractor/yahoo.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 031226f27..2d9e311b0 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -12,6 +12,7 @@ from ..compat import (
 )
 from ..utils import (
     clean_html,
+    unescapeHTML,
     ExtractorError,
     int_or_none,
 )
@@ -116,6 +117,16 @@ class YahooIE(InfoExtractor):
                 'description': 'md5:1428185051cfd1949807ad4ff6d3686a',
                 'duration': 201,
             }
+        }, {
+            'url': 'https://www.yahoo.com/movies/v/true-story-trailer-173000497.html',
+            'md5': '989396ae73d20c6f057746fb226aa215',
+            'info_dict': {
+                'id': '071c4013-ce30-3a93-a5b2-e0413cd4a9d1',
+                'ext': 'mp4',
+                'title': '\'True Story\' Trailer',
+                'description': 'True Story',
+                'duration': 150,
+            },
         }, {
             'url': 'https://gma.yahoo.com/pizza-delivery-man-surprised-huge-tip-college-kids-195200785.html',
             'only_matching': True,
@@ -125,6 +136,7 @@ class YahooIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         display_id = mobj.group('display_id')
+        page_id = mobj.group('id')
         url = mobj.group('url')
         host = mobj.group('host')
         webpage = self._download_webpage(url, display_id)
@@ -149,6 +161,7 @@ class YahooIE(InfoExtractor):
                 r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"',
                 r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
                 r'"first_videoid"\s*:\s*"([^"]+)"',
+                r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
             ]
             video_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
         else:
@@ -211,7 +224,7 @@ class YahooIE(InfoExtractor):
         return {
             'id': video_id,
             'display_id': display_id,
-            'title': meta['title'],
+            'title': unescapeHTML(meta['title']),
             'formats': formats,
             'description': clean_html(meta['description']),
             'thumbnail': meta['thumbnail'] if meta.get('thumbnail') else self._og_search_thumbnail(webpage),

From 85fab7e47b4d4d69a8d105e1d3a07d8a0dd67685 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 14:47:12 +0100
Subject: [PATCH 280/974] [yahoo] Replace two tests

The first one returned an internal server error.
The other doesn't seem to contain a video anymore.
---
 youtube_dl/extractor/yahoo.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 2d9e311b0..eed9d4325 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -56,14 +56,14 @@ class YahooIE(InfoExtractor):
             }
         },
         {
-            'url': 'https://tw.screen.yahoo.com/taipei-opinion-poll/選情站報-街頭民調-台北市篇-102823042.html',
-            'md5': '92a7fdd8a08783c68a174d7aa067dde8',
+            'url': 'https://tw.screen.yahoo.com/election-2014-askmayor/敢問市長-黃秀霜批賴清德-非常高傲-033009720.html',
+            'md5': '3a09cf59349cfaddae1797acc3c087fc',
             'info_dict': {
-                'id': '7a23b569-7bea-36cb-85b9-bd5301a0a1fb',
+                'id': 'cac903b3-fcf4-3c14-b632-643ab541712f',
                 'ext': 'mp4',
-                'title': '選情站報 街頭民調 台北市篇',
-                'description': '選情站報 街頭民調 台北市篇',
-                'duration': 429,
+                'title': '敢問市長／黃秀霜批賴清德「非常高傲」',
+                'description': '直言台南沒捷運 交通居五都之末',
+                'duration': 396,
             }
         },
         {
@@ -88,14 +88,14 @@ class YahooIE(InfoExtractor):
                 'duration': 121,
             }
         }, {
-            'url': 'https://ca.finance.yahoo.com/news/20-most-valuable-brands-world-112600775.html',
-            'md5': '3e401e4eed6325aa29d9b96125fd5b4f',
+            'url': 'https://ca.finance.yahoo.com/news/hackers-sony-more-trouble-well-154609075.html',
+            'md5': '226a895aae7e21b0129e2a2006fe9690',
             'info_dict': {
-                'id': 'c1b4c09c-8ed8-3b65-8b05-169c55358a83',
+                'id': 'e624c4bc-3389-34de-9dfc-025f74943409',
                 'ext': 'mp4',
-                'title': "Apple Is The World's Most Valuable Brand",
-                'description': 'md5:73eabc1a11c6f59752593b2ceefa1262',
-                'duration': 21,
+                'title': '\'The Interview\' TV Spot: War',
+                'description': 'The Interview',
+                'duration': 30,
             }
         }, {
             'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',

From 653d14e2f9bee934ca491123f0461a37f770e9e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 14:47:44 +0100
Subject: [PATCH 281/974] [yahoo] Update extraction process

Their webpage uses now https://video.media.yql.yahoo.com/v1/video/sapi/streams/ for getting the video info.
---
 youtube_dl/extractor/yahoo.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index eed9d4325..f8e7041a0 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -176,17 +176,15 @@ class YahooIE(InfoExtractor):
         region = self._search_regex(
             r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
             webpage, 'region', fatal=False, default='US')
-        query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"'
-                 ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="%s"'
-                 ' AND protocol="http"' % (video_id, region))
         data = compat_urllib_parse.urlencode({
-            'q': query,
-            'env': 'prod',
-            'format': 'json',
+            'protocol': 'http',
+            'region': region,
         })
+        query_url = (
+            'https://video.media.yql.yahoo.com/v1/video/sapi/streams/'
+            '{id}?{data}'.format(id=video_id, data=data))
         query_result = self._download_json(
-            'http://video.query.yahoo.com/v1/public/yql?' + data,
-            display_id, 'Downloading video info')
+            query_url, display_id, 'Downloading video info')
 
         info = query_result['query']['results']['mediaObj'][0]
         meta = info.get('meta')

From 1f809a856017dacc70462d743337b5197a0f980b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 15:22:30 +0100
Subject: [PATCH 282/974] [nerdcubed] Style fixes

---
 youtube_dl/extractor/nerdcubed.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nerdcubed.py b/youtube_dl/extractor/nerdcubed.py
index 9f2e678e0..efc903afa 100644
--- a/youtube_dl/extractor/nerdcubed.py
+++ b/youtube_dl/extractor/nerdcubed.py
@@ -18,7 +18,7 @@ class NerdCubedFeedIE(InfoExtractor):
 
     def _real_extract(self, url):
         feed = self._download_json(url, url, "Downloading NerdCubed JSON feed")
-        
+
         entries = [{
             '_type': 'url',
             'title': feed_entry['title'],
@@ -33,4 +33,3 @@ class NerdCubedFeedIE(InfoExtractor):
             'id': 'nerdcubed-feed',
             'entries': entries,
         }
-

From 3f1399de8ab73806aca5b418d1097feb5f06d091 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 15:26:00 +0100
Subject: [PATCH 283/974] [tmz] Fix test's thumbnail field

---
 youtube_dl/extractor/tmz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tmz.py b/youtube_dl/extractor/tmz.py
index 827aa08a4..c5c6fdc51 100644
--- a/youtube_dl/extractor/tmz.py
+++ b/youtube_dl/extractor/tmz.py
@@ -15,7 +15,7 @@ class TMZIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Kim Kardashian\'s Boobs Unlock a Mystery!',
             'description': 'Did Kim Kardasain try to one-up Khloe by one-upping Kylie???  Or is she just showing off her amazing boobs?',
-            'thumbnail': 'http://cdnbakmi.kaltura.com/p/591531/sp/59153100/thumbnail/entry_id/0_okj015ty/version/100002/acv/182/width/640',
+            'thumbnail': r're:http://cdnbakmi\.kaltura\.com/.*thumbnail.*',
         }
     }
 

From a86cbf587685230d73bb19ff619a8327fbcc29c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 15:28:40 +0100
Subject: [PATCH 284/974] [rtp] Fix test's id field

---
 youtube_dl/extractor/rtp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index 54f67a52f..7736cabba 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -12,7 +12,7 @@ class RTPIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
         'info_dict': {
-            'id': '174042',
+            'id': 'e174042',
             'ext': 'mp3',
             'title': 'Paixões Cruzadas',
             'description': 'As paixões musicais de António Cartaxo e António Macedo',

From 85d253af6b10dda05fa9d3821606346c3b26b697 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 15:35:36 +0100
Subject: [PATCH 285/974] [internetvideoarchive] Update test's duration field

---
 youtube_dl/extractor/internetvideoarchive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py
index c813d4b82..483cc6f9e 100644
--- a/youtube_dl/extractor/internetvideoarchive.py
+++ b/youtube_dl/extractor/internetvideoarchive.py
@@ -22,7 +22,7 @@ class InternetVideoArchiveIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'SKYFALL',
             'description': 'In SKYFALL, Bond\'s loyalty to M is tested as her past comes back to haunt her. As MI6 comes under attack, 007 must track down and destroy the threat, no matter how personal the cost.',
-            'duration': 149,
+            'duration': 152,
         },
     }
 

From 70f6796e7d644b78ed346163287a148cd72733e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 16:54:53 +0100
Subject: [PATCH 286/974] [telecinco] Rename 'episode' group to 'id' in the
 _VALID_URL regex

MiTeleIE now uses '_match_id'
---
 youtube_dl/extractor/telecinco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 2a2fff5e1..be3f72df7 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -6,7 +6,7 @@ from .mitele import MiTeleIE
 
 class TelecincoIE(MiTeleIE):
     IE_NAME = 'telecinco.es'
-    _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/[^/]+/(?P<episode>.*?)\.html'
+    _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/[^/]+/(?P<id>.*?)\.html'
 
     _TEST = {
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',

From c8f167823f483b4dd043e011722ccce9aa8223b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 16:57:07 +0100
Subject: [PATCH 287/974] [dbtv] Make sure the 'id' field is a string

---
 youtube_dl/extractor/dbtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dbtv.py b/youtube_dl/extractor/dbtv.py
index 1d3e2ff08..212217625 100644
--- a/youtube_dl/extractor/dbtv.py
+++ b/youtube_dl/extractor/dbtv.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     float_or_none,
     int_or_none,
@@ -61,7 +62,7 @@ class DBTVIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video['id'],
+            'id': compat_str(video['id']),
             'display_id': display_id,
             'title': video['title'],
             'description': clean_html(video['desc']),

From 5d3808524ddcb9e4b2f4cadf69fc590b1bd0a405 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 16:58:29 +0100
Subject: [PATCH 288/974] [extractor/common] Update docstring: replace
 FileDownloader with YoutubeDL

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d302fe45f..4b950e485 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -40,7 +40,7 @@ class InfoExtractor(object):
     information about the video (or videos) the URL refers to. This
     information includes the real video URL, the video title, author and
     others. The information is stored in a dictionary which is then
-    passed to the FileDownloader. The FileDownloader processes this
+    passed to the YoutubeDL. The YoutubeDL processes this
     information possibly downloading the video to the file system, among
     other possible outcomes.
 

From a8896c5ac202c17f60d5d68cb70f73ceceee52d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Dec 2014 00:58:15 +0600
Subject: [PATCH 289/974] [crunchyroll] Add .fr domain (#4537)

---
 youtube_dl/extractor/crunchyroll.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 2c7756eb6..354046a9e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -29,8 +29,8 @@ from .common import InfoExtractor
 
 
 class CrunchyrollIE(SubtitlesInfoExtractor):
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
+    _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
         'info_dict': {
             'id': '645513',
@@ -46,7 +46,10 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
             # rtmp
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
+        'only_matching': True,
+    }]
 
     _FORMAT_IDS = {
         '360': ('60', '106'),

From f00fd51dae685cee2406401b50b847878e9a9d2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 20:49:14 +0100
Subject: [PATCH 290/974] Don't write the description file if
 info_dict['description'] is None (#3166)

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 14ee30e55..e2b823f66 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1016,13 +1016,13 @@ class YoutubeDL(object):
             descfn = filename + '.description'
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
                 self.to_screen('[info] Video description is already present')
+            elif info_dict.get('description') is None:
+                self.report_warning('There\'s no description to write.')
             else:
                 try:
                     self.to_screen('[info] Writing video description to: ' + descfn)
                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
                         descfile.write(info_dict['description'])
-                except (KeyError, TypeError):
-                    self.report_warning('There\'s no description to write.')
                 except (OSError, IOError):
                     self.report_error('Cannot write description file ' + descfn)
                     return

From 1362bbbb4bfbd9380211a66032dbe3fe09aa2173 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Dec 2014 22:05:47 +0600
Subject: [PATCH 291/974] [adobetv] Add extractor (Closes #4536)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/adobetv.py  | 70 ++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 youtube_dl/extractor/adobetv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 76f13bf52..92aca503c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from .abc import ABCIE
 from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
+from .adobetv import AdobeTVIE
 from .adultswim import AdultSwimIE
 from .aftonbladet import AftonbladetIE
 from .aljazeera import AlJazeeraIE
diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
new file mode 100644
index 000000000..28e07f8b0
--- /dev/null
+++ b/youtube_dl/extractor/adobetv.py
@@ -0,0 +1,70 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    unified_strdate,
+    str_to_int,
+)
+
+
+class AdobeTVIE(InfoExtractor):
+    _VALID_URL = r'https?://tv\.adobe\.com/watch/[^/]+/(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://tv.adobe.com/watch/the-complete-picture-with-julieanne-kost/quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop/',
+        'md5': '9bc5727bcdd55251f35ad311ca74fa1e',
+        'info_dict': {
+            'id': 'quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop',
+            'ext': 'mp4',
+            'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop',
+            'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'upload_date': '20110914',
+            'duration': 60,
+            'view_count': int,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        player = self._parse_json(
+            self._search_regex(r'html5player:\s*({.+?})\s*\n', webpage, 'player'),
+            video_id)
+
+        title = player.get('title') or self._search_regex(
+            r'data-title="([^"]+)"', webpage, 'title')
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        upload_date = unified_strdate(
+            self._html_search_meta('datepublished', webpage, 'upload date'))
+
+        duration = parse_duration(
+            self._html_search_meta('duration', webpage, 'duration')
+            or self._search_regex(r'Runtime:\s*(\d{2}:\d{2}:\d{2})', webpage, 'duration'))
+
+        view_count = str_to_int(self._search_regex(
+            r'<div class="views">\s*Views?:\s*([\d,.]+)\s*</div>',
+            webpage, 'view count'))
+
+        formats = [{
+            'url': source['src'],
+            'format_id': source.get('quality') or source['src'].split('-')[-1].split('.')[0] or None,
+            'tbr': source.get('bitrate'),
+        } for source in player['sources']]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'view_count': view_count,
+            'formats': formats,
+        }

From 9532d723715f1d1a08c126ba8af6940113b88d3b Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Mon, 22 Dec 2014 18:40:36 +0100
Subject: [PATCH 292/974] [cnn] Add support for articles with videos (fixes
 #4541)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/cnn.py      | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 92aca503c..ba12e3263 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -64,6 +64,7 @@ from .cnet import CNETIE
 from .cnn import (
     CNNIE,
     CNNBlogsIE,
+    CNNArticleIE,
 )
 from .collegehumor import CollegeHumorIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 81142ee41..cddcacdf1 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -127,3 +127,28 @@ class CNNBlogsIE(InfoExtractor):
             'url': cnn_url,
             'ie_key': CNNIE.ie_key(),
         }
+
+
+class CNNArticleIE(InfoExtractor):
+    _VALID_URL = r'https?://((edition|www)\.)?cnn\.com/(?!video/)'
+    _TEST = {
+        'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
+        'md5': '275b326f85d80dff7592a9820f5dc887',
+        'info_dict': {
+            'id': 'bestoftv/2014/12/21/sotu-crowley-president-obama-north-korea-not-going-to-be-intimidated.cnn',
+            'ext': 'mp4',
+            'title': 'Obama: We\'re not going to be intimidated',
+            'description': 'md5:e735586f3dc936075fa654a4d91b21f9',
+            'upload_date': '20141220',
+        },
+        'add_ie': ['CNN'],
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, url_basename(url))
+        cnn_url = self._html_search_regex(r"video:\s*'([^']+)'", webpage, 'cnn url')
+        return {
+            '_type': 'url',
+            'url': r'http://cnn.com/video/?/video/' + cnn_url,
+            'ie_key': CNNIE.ie_key(),
+        }

From e0ae1814b112b100b73957fe1349d1569ec7a3dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 22 Dec 2014 22:24:19 +0100
Subject: [PATCH 293/974] [sportdeutschland] Fix extraction (fixes #4544)

---
 youtube_dl/extractor/sportdeutschland.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index 2f57f5b7c..1a57aebf1 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -60,9 +60,10 @@ class SportDeutschlandIE(InfoExtractor):
 
         categories = list(data.get('section', {}).get('tags', {}).values())
         asset = data['asset']
+        assets_info = self._download_json(asset['url'], video_id)
 
         formats = []
-        smil_url = asset['video']
+        smil_url = assets_info['video']
         if '.smil' in smil_url:
             m3u8_url = smil_url.replace('.smil', '.m3u8')
             formats.extend(

From 544dec62980f4e5733747c175b64c2837d5f6c0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 Dec 2014 20:33:56 +0600
Subject: [PATCH 294/974] [smotri] Skip broken tests

---
 youtube_dl/extractor/smotri.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index d031fe401..baef3daa0 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -69,6 +69,7 @@ class SmotriIE(InfoExtractor):
             'params': {
                 'videopassword': 'qwerty',
             },
+            'skip': 'Video is not approved by moderator',
         },
         # age limit + video-password
         {
@@ -86,7 +87,8 @@ class SmotriIE(InfoExtractor):
             },
             'params': {
                 'videopassword': '333'
-            }
+            },
+            'skip': 'Video is not approved by moderator',
         },
         # swf player
         {

From 55f0cab3a38135060c967023accb5b5f3fa3dfb5 Mon Sep 17 00:00:00 2001
From: Tobias Bell <tobias.bell@gmail.com>
Date: Tue, 23 Dec 2014 22:09:21 +0100
Subject: [PATCH 295/974] [gameone] This fix resolves issue #4552

The duration metadata for certain episodes contained floating point
numbers instead of integers. Now only the integer part will be
interpreted. Also added a test for this
---
 youtube_dl/extractor/gameone.py | 50 ++++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/gameone.py b/youtube_dl/extractor/gameone.py
index 3022f539d..da9d44791 100644
--- a/youtube_dl/extractor/gameone.py
+++ b/youtube_dl/extractor/gameone.py
@@ -21,21 +21,38 @@ RAW_MP4_URL = 'http://cdn.riptide-mtvn.com/'
 
 class GameOneIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gameone\.de/tv/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.gameone.de/tv/288',
-        'md5': '136656b7fb4c9cb4a8e2d500651c499b',
-        'info_dict': {
-            'id': '288',
-            'ext': 'mp4',
-            'title': 'Game One - Folge 288',
-            'duration': 1238,
-            'thumbnail': 'http://s3.gameone.de/gameone/assets/video_metas/teaser_images/000/643/636/big/640x360.jpg',
-            'description': 'FIFA-Pressepokal 2014, Star Citizen, Kingdom Come: Deliverance, Project Cars, Schöner Trants Nerdquiz Folge 2 Runde 1',
-            'age_limit': 16,
-            'upload_date': '20140513',
-            'timestamp': 1399980122,
+    _TESTS = [
+        {
+            'url': 'http://www.gameone.de/tv/288',
+            'md5': '136656b7fb4c9cb4a8e2d500651c499b',
+            'info_dict': {
+                'id': '288',
+                'ext': 'mp4',
+                'title': 'Game One - Folge 288',
+                'duration': 1238,
+                'thumbnail': 'http://s3.gameone.de/gameone/assets/video_metas/teaser_images/000/643/636/big/640x360.jpg',
+                'description': 'FIFA-Pressepokal 2014, Star Citizen, Kingdom Come: Deliverance, Project Cars, Schöner Trants Nerdquiz Folge 2 Runde 1',
+                'age_limit': 16,
+                'upload_date': '20140513',
+                'timestamp': 1399980122,
+            }
+        },
+        {
+            'url': 'http://gameone.de/tv/220',
+            'md5': '5227ca74c4ae6b5f74c0510a7c48839e',
+            'info_dict': {
+                'id': '220',
+                'ext': 'mp4',
+                'upload_date': '20120918',
+                'description': 'Jet Set Radio HD, Tekken Tag Tournament 2, Source Filmmaker',
+                'timestamp': 1347971451,
+                'title': 'Game One - Folge 220',
+                'duration': 896,
+                'age_limit': 16,
+            }
         }
-    }
+
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -66,7 +83,7 @@ class GameOneIE(InfoExtractor):
             video_id,
             'Downloading media:content')
         rendition_items = content.findall('.//rendition')
-        duration = int(rendition_items[0].get('duration'))
+        duration = int(rendition_items[0].get('duration').split('.')[0])
         formats = [
             {
                 'url': re.sub(r'.*/(r2)', RAW_MP4_URL + r'\1', r.find('./src').text),
@@ -105,7 +122,8 @@ class GameOnePlaylistIE(InfoExtractor):
         webpage = self._download_webpage('http://www.gameone.de/tv', 'TV')
         max_id = max(map(int, re.findall(r'<a href="/tv/(\d+)"', webpage)))
         entries = [
-            self.url_result('http://www.gameone.de/tv/%d' % video_id, 'GameOne')
+            self.url_result('http://www.gameone.de/tv/%d' %
+                            video_id, 'GameOne')
             for video_id in range(max_id, 0, -1)]
 
         return {

From 39ac7c94354967b44588155b2d6752f780eaaf16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Dec 2014 19:18:59 +0600
Subject: [PATCH 296/974] [gameone] Extract duration as float

---
 youtube_dl/extractor/gameone.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/gameone.py b/youtube_dl/extractor/gameone.py
index da9d44791..75f180928 100644
--- a/youtube_dl/extractor/gameone.py
+++ b/youtube_dl/extractor/gameone.py
@@ -6,7 +6,9 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     xpath_with_ns,
-    parse_iso8601
+    parse_iso8601,
+    float_or_none,
+    int_or_none,
 )
 
 NAMESPACE_MAP = {
@@ -47,7 +49,7 @@ class GameOneIE(InfoExtractor):
                 'description': 'Jet Set Radio HD, Tekken Tag Tournament 2, Source Filmmaker',
                 'timestamp': 1347971451,
                 'title': 'Game One - Folge 220',
-                'duration': 896,
+                'duration': 896.62,
                 'age_limit': 16,
             }
         }
@@ -83,13 +85,13 @@ class GameOneIE(InfoExtractor):
             video_id,
             'Downloading media:content')
         rendition_items = content.findall('.//rendition')
-        duration = int(rendition_items[0].get('duration').split('.')[0])
+        duration = float_or_none(rendition_items[0].get('duration'))
         formats = [
             {
                 'url': re.sub(r'.*/(r2)', RAW_MP4_URL + r'\1', r.find('./src').text),
-                'width': int(r.get('width')),
-                'height': int(r.get('height')),
-                'tbr': int(r.get('bitrate')),
+                'width': int_or_none(r.get('width')),
+                'height': int_or_none(r.get('height')),
+                'tbr': int_or_none(r.get('bitrate')),
             }
             for r in rendition_items
         ]

From 5ac71f0b27fbbca11517e5592978a2fe7bfd9f34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Dec 2014 22:25:05 +0600
Subject: [PATCH 297/974] [sohu] Modernize and extract all formats and more
 metadata (Closes #4409, closes #2056, closes #3009)

---
 youtube_dl/extractor/sohu.py | 92 ++++++++++++++++++++----------------
 1 file changed, 52 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 07f514a46..c04791997 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -1,11 +1,10 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from .common import compat_str
 
 
 class SohuIE(InfoExtractor):
@@ -29,60 +28,73 @@ class SohuIE(InfoExtractor):
                 base_data_url = 'http://my.tv.sohu.com/play/videonew.do?vid='
             else:
                 base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
-            data_url = base_data_url + str(vid_id)
-            data_json = self._download_webpage(
-                data_url, video_id,
-                note='Downloading JSON data for ' + str(vid_id))
-            return json.loads(data_json)
+
+            return self._download_json(
+                base_data_url + vid_id, video_id,
+                'Downloading JSON data for %s' % vid_id)
 
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         mytv = mobj.group('mytv') is not None
 
         webpage = self._download_webpage(url, video_id)
-        raw_title = self._html_search_regex(r'(?s)<title>(.+?)</title>',
-                                            webpage, 'video title')
+        raw_title = self._html_search_regex(
+            r'(?s)<title>(.+?)</title>',
+            webpage, 'video title')
         title = raw_title.partition('-')[0].strip()
 
-        vid = self._html_search_regex(r'var vid ?= ?["\'](\d+)["\']', webpage,
-                                      'video path')
-        data = _fetch_data(vid, mytv)
+        vid = self._html_search_regex(
+            r'var vid ?= ?["\'](\d+)["\']',
+            webpage, 'video path')
+        vid_data = _fetch_data(vid, mytv)
 
-        QUALITIES = ('ori', 'super', 'high', 'nor')
-        vid_ids = [data['data'][q + 'Vid']
-                   for q in QUALITIES
-                   if data['data'][q + 'Vid'] != 0]
-        if not vid_ids:
-            raise ExtractorError('No formats available for this video')
+        formats_json = {}
+        for format_id in ('nor', 'high', 'super', 'ori', 'h2644k', 'h2654k'):
+            vid_id = vid_data['data'].get('%sVid' % format_id)
+            if not vid_id:
+                continue
+            vid_id = compat_str(vid_id)
+            formats_json[format_id] = vid_data if vid == vid_id else _fetch_data(vid_id, mytv)
 
-        # For now, we just pick the highest available quality
-        vid_id = vid_ids[-1]
-
-        format_data = data if vid == vid_id else _fetch_data(vid_id, mytv)
-        part_count = format_data['data']['totalBlocks']
-        allot = format_data['allot']
-        prot = format_data['prot']
-        clipsURL = format_data['data']['clipsURL']
-        su = format_data['data']['su']
+        part_count = vid_data['data']['totalBlocks']
 
         playlist = []
         for i in range(part_count):
-            part_url = ('http://%s/?prot=%s&file=%s&new=%s' %
-                        (allot, prot, clipsURL[i], su[i]))
-            part_str = self._download_webpage(
-                part_url, video_id,
-                note='Downloading part %d of %d' % (i + 1, part_count))
+            formats = []
+            for format_id, format_data in formats_json.items():
+                allot = format_data['allot']
+                prot = format_data['prot']
 
-            part_info = part_str.split('|')
-            video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
+                data = format_data['data']
+                clips_url = data['clipsURL']
+                su = data['su']
 
-            video_info = {
-                'id': '%s_part%02d' % (video_id, i + 1),
+                part_str = self._download_webpage(
+                    'http://%s/?prot=%s&file=%s&new=%s' %
+                    (allot, prot, clips_url[i], su[i]),
+                    video_id,
+                    'Downloading %s video URL part %d of %d'
+                    % (format_id, i + 1, part_count))
+
+                part_info = part_str.split('|')
+                video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
+
+                formats.append({
+                    'url': video_url,
+                    'format_id': format_id,
+                    'filesize': data['clipsBytes'][i],
+                    'width': data['width'],
+                    'height': data['height'],
+                    'fps': data['fps'],
+                })
+            self._sort_formats(formats)
+
+            playlist.append({
+                'id': '%s_part%d' % (video_id, i + 1),
                 'title': title,
-                'url': video_url,
-                'ext': 'mp4',
-            }
-            playlist.append(video_info)
+                'duration': vid_data['data']['clipsDuration'][i],
+                'formats': formats,
+            })
 
         if len(playlist) == 1:
             info = playlist[0]

From 885e4384a1ad8b12037810824d15e9436134a148 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Dec 2014 23:26:57 +0600
Subject: [PATCH 298/974] [teletask] Simplify

---
 youtube_dl/extractor/teletask.py | 50 ++++++++++++--------------------
 1 file changed, 18 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py
index aa5535042..e54145105 100644
--- a/youtube_dl/extractor/teletask.py
+++ b/youtube_dl/extractor/teletask.py
@@ -1,13 +1,13 @@
-# coding: utf-8
 from __future__ import unicode_literals
+
 import re
-import datetime
 
 from .common import InfoExtractor
+from ..utils import unified_strdate
 
 
 class TeleTaskIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://(?:www\.)?tele-task\.de/archive/video/html5/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.tele-task.de/archive/video/html5/26168/',
         'info_dict': {
@@ -16,52 +16,38 @@ class TeleTaskIE(InfoExtractor):
         'playlist': [{
             'md5': '290ef69fb2792e481169c3958dbfbd57',
             'info_dict': {
+                'id': '26168-speaker',
+                'ext': 'mp4',
                 'title': 'Duplicate Detection',
                 'upload_date': '20141218',
-                'id': 'speaker_26168',
-                'ext': 'mp4',
             }
-        },
-            {
+        }, {
             'md5': 'e1e7218c5f0e4790015a437fcf6c71b4',
             'info_dict': {
+                'id': '26168-slides',
+                'ext': 'mp4',
                 'title': 'Duplicate Detection',
                 'upload_date': '20141218',
-                'id': 'slides_26168',
-                'ext': 'mp4',
             }
         }]
     }
 
     def _real_extract(self, url):
         lecture_id = self._match_id(url)
+
         webpage = self._download_webpage(url, lecture_id)
 
         title = self._html_search_regex(
-            r'itemprop="name">([^"]+)</a>', webpage, 'title')
-        url_speaker = self._html_search_regex(
-            r'class="speaker".*?src="([^"]+)"', webpage, 'video_url_speaker', flags=re.DOTALL)
-        url_slides = self._html_search_regex(
-            r'class="slides".*?src="([^"]+)"', webpage, 'video_url_slides', flags=re.DOTALL)
-        date = self._html_search_regex(
-            r'<td class="label">Date:</td><td>([^"]+)</td>', webpage, 'date')
-        date = datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y%m%d')
+            r'itemprop="name">([^<]+)</a>', webpage, 'title')
+        upload_date = unified_strdate(self._html_search_regex(
+            r'Date:</td><td>([^<]+)</td>', webpage, 'date', fatal=False))
 
         entries = [{
+            'id': '%s-%s' % (lecture_id, format_id),
+            'url': video_url,
             'title': title,
-            'upload_date': date,
-            'id': "speaker_"+lecture_id,
-            'url': url_speaker,
-        },
-            {
-            'title': title,
-            'upload_date': date,
-            'id': "slides_"+lecture_id,
-            'url': url_slides}]
+            'upload_date': upload_date,
+        } for format_id, video_url in re.findall(
+            r'<video class="([^"]+)"[^>]*>\s*<source src="([^"]+)"', webpage)]
 
-        return {
-            '_type': "playlist",
-            'id': lecture_id,
-            'title': title,
-            'entries': entries,
-        }
+        return self.playlist_result(entries, lecture_id, title)

From a5424052000d6ab69412f380a96f1963f315a295 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Dec 2014 23:29:10 +0600
Subject: [PATCH 299/974] Credit @MaxReimann for teletask (#4533)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 6ea958fce..bb4d8b4d1 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -95,3 +95,4 @@ Adrian Kretz
 Mathias Rav
 Petr Kutalek
 Will Glynn
+Max Reimann

From 2c0b4752357b0102220874040934d0d124bed35b Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Fri, 26 Dec 2014 12:49:13 +0100
Subject: [PATCH 300/974] [sunporno] fix duration

---
 youtube_dl/extractor/sunporno.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 263f09b46..6cfbc0160 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -39,7 +39,7 @@ class SunPornoIE(InfoExtractor):
             r'poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
 
         duration = parse_duration(self._search_regex(
-            r'Duration:\s*(\d+:\d+)\s*<', webpage, 'duration', fatal=False))
+            r'Duration:[^\d]+(\d+:\d+)\s*<', webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._html_search_regex(
             r'class="views">\s*(\d+)\s*<', webpage, 'view count', fatal=False))

From 31424c126fb48acc49cd539bad6aa7193c7583d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Dec 2014 19:28:51 +0600
Subject: [PATCH 301/974] [sunporno] Modernize

---
 youtube_dl/extractor/sunporno.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 6cfbc0160..8a333f1d2 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -28,23 +28,27 @@ class SunPornoIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
-        description = self._html_search_meta('description', webpage, 'description')
+        title = self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title')
+        description = self._html_search_meta(
+            'description', webpage, 'description')
         thumbnail = self._html_search_regex(
             r'poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
 
         duration = parse_duration(self._search_regex(
-            r'Duration:[^\d]+(\d+:\d+)\s*<', webpage, 'duration', fatal=False))
+            r'itemprop="duration">\s*(\d+:\d+)\s*<',
+            webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._html_search_regex(
-            r'class="views">\s*(\d+)\s*<', webpage, 'view count', fatal=False))
+            r'class="views">\s*(\d+)\s*<',
+            webpage, 'view count', fatal=False))
         comment_count = int_or_none(self._html_search_regex(
-            r'(\d+)</b> Comments?', webpage, 'comment count', fatal=False))
+            r'(\d+)</b> Comments?',
+            webpage, 'comment count', fatal=False))
 
         formats = []
         quality = qualities(['mp4', 'flv'])

From 6a1c4fbfcb57c138b8ace305127c2654844e8099 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Fri, 26 Dec 2014 15:49:12 +0100
Subject: [PATCH 302/974] [hellporno] new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/hellporno.py | 56 +++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/hellporno.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fd0ebffe3..de5206729 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -166,6 +166,7 @@ from .grooveshark import GroovesharkIE
 from .groupon import GrouponIE
 from .hark import HarkIE
 from .heise import HeiseIE
+from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
 from .hornbunny import HornBunnyIE
diff --git a/youtube_dl/extractor/hellporno.py b/youtube_dl/extractor/hellporno.py
new file mode 100644
index 000000000..06f85127f
--- /dev/null
+++ b/youtube_dl/extractor/hellporno.py
@@ -0,0 +1,56 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+class HellPornoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hellporno\.com/videos/(?P<display_id>[^/]+)'
+    _TEST = {
+        'url': 'http://hellporno.com/videos/dixie-is-posing-with-naked-ass-very-erotic/',
+        'md5': '1fee339c610d2049699ef2aa699439f1',
+        'info_dict': {
+            'id': '149116',
+            'ext': 'mp4',
+            'title': 'Dixie is posing with naked ass very erotic',
+            'description': 'md5:5ba02cbf31eff820147b3cc25306d89a',
+            'categories': list,  # NSFW
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, 'main')
+
+        video_id = self._html_search_regex(r'video_id:\s*\'([^\']+)\'', webpage, 'id')
+
+        video_url = self._html_search_regex(r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
+
+        ext = self._html_search_regex(r'postfix:\s*\'([^\']+)\'', webpage, 'ext')[1:]
+
+        title = self._html_search_regex(
+            r'<title>([^<]+)\s*-\s*Hell Porno</title>', webpage, 'title')
+
+        description = self._html_search_meta('description', webpage, 'description', fatal=False)
+
+        thumbnail = self._html_search_regex(
+            r'preview_url:\s*\'([^\']+)\'',
+            webpage, 'thumbnail', fatal=False)
+
+        categories_str = self._html_search_regex(
+            r'<meta name="keywords" content="([^"]+)"', webpage, 'categories', fatal=False)
+        categories = (
+            None if categories_str is None
+            else categories_str.split(','))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': ext,
+            'description': description,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'age_limit': 18,
+        }

From 4cda41ac7be1af03cb5fc4e9bbfcbf944925ab63 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Fri, 26 Dec 2014 16:17:35 +0100
Subject: [PATCH 303/974] [alphaporno] new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/alphaporno.py | 54 ++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 youtube_dl/extractor/alphaporno.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fd0ebffe3..3fd0240c3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -7,6 +7,7 @@ from .adobetv import AdobeTVIE
 from .adultswim import AdultSwimIE
 from .aftonbladet import AftonbladetIE
 from .aljazeera import AlJazeeraIE
+from .alphaporno import AlphaPornoIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
 from .aol import AolIE
diff --git a/youtube_dl/extractor/alphaporno.py b/youtube_dl/extractor/alphaporno.py
new file mode 100644
index 000000000..c8eaf837f
--- /dev/null
+++ b/youtube_dl/extractor/alphaporno.py
@@ -0,0 +1,54 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+class AlphaPornoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?alphaporno\.com/videos/(?P<display_id>[^/]+)'
+    _TEST = {
+        'url': 'http://www.alphaporno.com/videos/sensual-striptease-porn-with-samantha-alexandra/',
+        'md5': 'feb6d3bba8848cd54467a87ad34bd38e',
+        'info_dict': {
+            'id': '258807',
+            'ext': 'mp4',
+            'title': 'Sensual striptease porn with Samantha Alexandra - Striptease Porn',
+            'description': 'md5:c4447dc80e5be4c5f2711f7806e45424',
+            'categories': list,  # NSFW
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, 'main')
+
+        video_id = self._html_search_regex(r'video_id:\s*\'([^\']+)\'', webpage, 'id')
+
+        video_url = self._html_search_regex(r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
+
+        ext = self._html_search_meta('encodingFormat', webpage, 'ext')[1:]
+
+        title = self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title')
+
+        description = self._html_search_meta('description', webpage, 'description', fatal=False)
+
+        thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail', fatal=False)
+
+        categories_str = self._html_search_meta(
+            'keywords', webpage, 'categories', fatal=False)
+        categories = (
+            None if categories_str is None
+            else categories_str.split(','))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': ext,
+            'description': description,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'age_limit': 18,
+        }

From 461b00f34aabf6e75e0d53419919a8e6cc9cfd7a Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Fri, 26 Dec 2014 17:15:34 +0100
Subject: [PATCH 304/974] [eroprofile] new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/eroprofile.py | 53 ++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/eroprofile.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fd0ebffe3..39788712c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -108,6 +108,7 @@ from .elpais import ElPaisIE
 from .empflix import EMPFlixIE
 from .engadget import EngadgetIE
 from .eporner import EpornerIE
+from .eroprofile import EroProfileIE
 from .escapist import EscapistIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
new file mode 100644
index 000000000..f6745fc52
--- /dev/null
+++ b/youtube_dl/extractor/eroprofile.py
@@ -0,0 +1,53 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+class EroProfileIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?eroprofile\.com/m/videos/view/(?P<display_id>[^/]+)'
+    _TEST = {
+        'url': 'http://www.eroprofile.com/m/videos/view/sexy-babe-softcore',
+        'md5': 'c26f351332edf23e1ea28ce9ec9de32f',
+        'info_dict': {
+            'id': '3733775',
+            'ext': 'm4v',
+            'title': 'HD MOVIES - sexy babe softcore',
+            'description': 'md5:831ee50526c333eb4f6c1e58d382c295',
+            'categories': list,  # NSFW
+            'thumbnail': 're:https?://.*\.jpg',
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, 'main')
+
+        video_id = self._html_search_regex(r'glbUpdViews\s*\(\'\d*\',\'(\d+)\'', webpage, 'id')
+
+        video_url = self._html_search_regex(r'<source src="([^"]+)', webpage, 'video_url')
+
+        title = self._html_search_regex(
+            r'<title>([^<]+)\s*-\s*EroProfile</title>', webpage, 'title')
+
+        description = self._html_search_meta('description', webpage, 'description', fatal=False)
+
+        thumbnail = self._html_search_regex(
+          r'onclick="showVideoPlayer\(\)"><img src="([^"]+)', webpage, 'thumbnail', fatal=False)
+
+        categories_str = self._html_search_meta(
+            'keywords', webpage, 'categories', fatal=False)
+        categories = (
+            None if categories_str is None
+            else categories_str.split(','))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': 'm4v',
+            'description': description,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'age_limit': 18,
+        }

From f74b341dde36fe2f56ab52c92416eb3ff5ccecf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Dec 2014 23:07:24 +0600
Subject: [PATCH 305/974] expect_info_dict actual-expected argument consistency

---
 test/helper.py        | 2 +-
 test/test_download.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 8a820526a..96d58b7c1 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -99,7 +99,7 @@ def gettestcases(include_onlymatching=False):
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 
-def expect_info_dict(self, expected_dict, got_dict):
+def expect_info_dict(self, got_dict, expected_dict):
     for info_field, expected in expected_dict.items():
         if isinstance(expected, compat_str) and expected.startswith('re:'):
             got = got_dict.get(info_field)
diff --git a/test/test_download.py b/test/test_download.py
index a009aa475..412f3dbce 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -155,7 +155,7 @@ def generator(test_case):
             if is_playlist:
                 self.assertEqual(res_dict['_type'], 'playlist')
                 self.assertTrue('entries' in res_dict)
-                expect_info_dict(self, test_case.get('info_dict', {}), res_dict)
+                expect_info_dict(self, res_dict, test_case.get('info_dict', {}))
 
             if 'playlist_mincount' in test_case:
                 assertGreaterEqual(
@@ -204,7 +204,7 @@ def generator(test_case):
                 with io.open(info_json_fn, encoding='utf-8') as infof:
                     info_dict = json.load(infof)
 
-                expect_info_dict(self, tc.get('info_dict', {}), info_dict)
+                expect_info_dict(self, info_dict, tc.get('info_dict', {}))
         finally:
             try_rm_tcs_files()
             if is_playlist and res_dict is not None and res_dict.get('entries'):

From f9b9e88646fcad16501a8390c040f9dd2257dd15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Dec 2014 23:32:41 +0600
Subject: [PATCH 306/974] [eroprofile] Simplify

---
 youtube_dl/extractor/eroprofile.py | 40 ++++++++++++------------------
 1 file changed, 16 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index f6745fc52..79e2fbd39 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -1,53 +1,45 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
+
 class EroProfileIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?eroprofile\.com/m/videos/view/(?P<display_id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?eroprofile\.com/m/videos/view/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://www.eroprofile.com/m/videos/view/sexy-babe-softcore',
         'md5': 'c26f351332edf23e1ea28ce9ec9de32f',
         'info_dict': {
             'id': '3733775',
+            'display_id': 'sexy-babe-softcore',
             'ext': 'm4v',
-            'title': 'HD MOVIES - sexy babe softcore',
-            'description': 'md5:831ee50526c333eb4f6c1e58d382c295',
-            'categories': list,  # NSFW
+            'title': 'sexy babe softcore',
             'thumbnail': 're:https?://.*\.jpg',
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        webpage = self._download_webpage(url, 'main')
+        display_id = self._match_id(url)
 
-        video_id = self._html_search_regex(r'glbUpdViews\s*\(\'\d*\',\'(\d+)\'', webpage, 'id')
+        webpage = self._download_webpage(url, display_id)
 
-        video_url = self._html_search_regex(r'<source src="([^"]+)', webpage, 'video_url')
+        video_id = self._search_regex(
+            [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],
+            webpage, 'video id', default=None)
 
+        video_url = self._search_regex(
+            r'<source src="([^"]+)', webpage, 'video url')
         title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*EroProfile</title>', webpage, 'title')
-
-        description = self._html_search_meta('description', webpage, 'description', fatal=False)
-
-        thumbnail = self._html_search_regex(
-          r'onclick="showVideoPlayer\(\)"><img src="([^"]+)', webpage, 'thumbnail', fatal=False)
-
-        categories_str = self._html_search_meta(
-            'keywords', webpage, 'categories', fatal=False)
-        categories = (
-            None if categories_str is None
-            else categories_str.split(','))
+            r'Title:</th><td>([^<]+)</td>', webpage, 'title')
+        thumbnail = self._search_regex(
+            r'onclick="showVideoPlayer\(\)"><img src="([^"]+)',
+            webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'url': video_url,
             'title': title,
-            'ext': 'm4v',
-            'description': description,
             'thumbnail': thumbnail,
-            'categories': categories,
             'age_limit': 18,
         }

From e82def52a97faf85eeeedcac1266c27a565e2f21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Dec 2014 00:08:04 +0600
Subject: [PATCH 307/974] [alphaporno] Improve

---
 youtube_dl/extractor/alphaporno.py | 71 ++++++++++++++++++++----------
 1 file changed, 47 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/alphaporno.py b/youtube_dl/extractor/alphaporno.py
index c8eaf837f..c34719d1f 100644
--- a/youtube_dl/extractor/alphaporno.py
+++ b/youtube_dl/extractor/alphaporno.py
@@ -1,54 +1,77 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    parse_duration,
+    parse_filesize,
+    int_or_none,
+)
+
 
 class AlphaPornoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?alphaporno\.com/videos/(?P<display_id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?alphaporno\.com/videos/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://www.alphaporno.com/videos/sensual-striptease-porn-with-samantha-alexandra/',
         'md5': 'feb6d3bba8848cd54467a87ad34bd38e',
         'info_dict': {
             'id': '258807',
+            'display_id': 'sensual-striptease-porn-with-samantha-alexandra',
             'ext': 'mp4',
-            'title': 'Sensual striptease porn with Samantha Alexandra - Striptease Porn',
-            'description': 'md5:c4447dc80e5be4c5f2711f7806e45424',
-            'categories': list,  # NSFW
+            'title': 'Sensual striptease porn with Samantha Alexandra',
             'thumbnail': 're:https?://.*\.jpg$',
+            'timestamp': 1418694611,
+            'upload_date': '20141216',
+            'duration': 387,
+            'filesize_approx': 54120000,
+            'tbr': 1145,
+            'categories': list,
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        webpage = self._download_webpage(url, 'main')
+        display_id = self._match_id(url)
 
-        video_id = self._html_search_regex(r'video_id:\s*\'([^\']+)\'', webpage, 'id')
+        webpage = self._download_webpage(url, display_id)
 
-        video_url = self._html_search_regex(r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
+        video_id = self._search_regex(
+            r"video_id\s*:\s*'([^']+)'", webpage, 'video id', default=None)
 
-        ext = self._html_search_meta('encodingFormat', webpage, 'ext')[1:]
+        video_url = self._search_regex(
+            r"video_url\s*:\s*'([^']+)'", webpage, 'video url')
+        ext = self._html_search_meta(
+            'encodingFormat', webpage, 'ext', default='.mp4')[1:]
 
-        title = self._html_search_regex(
-            r'<title>([^<]+)</title>', webpage, 'title')
+        title = self._search_regex(
+            [r'<meta content="([^"]+)" itemprop="description">',
+             r'class="title" itemprop="name">([^<]+)<'],
+            webpage, 'title')
+        thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail')
+        timestamp = parse_iso8601(self._html_search_meta(
+            'uploadDate', webpage, 'upload date'))
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration'))
+        filesize_approx = parse_filesize(self._html_search_meta(
+            'contentSize', webpage, 'file size'))
+        bitrate = int_or_none(self._html_search_meta(
+            'bitrate', webpage, 'bitrate'))
+        categories = self._html_search_meta(
+            'keywords', webpage, 'categories', default='').split(',')
 
-        description = self._html_search_meta('description', webpage, 'description', fatal=False)
-
-        thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail', fatal=False)
-
-        categories_str = self._html_search_meta(
-            'keywords', webpage, 'categories', fatal=False)
-        categories = (
-            None if categories_str is None
-            else categories_str.split(','))
+        age_limit = self._rta_search(webpage)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'url': video_url,
-            'title': title,
             'ext': ext,
-            'description': description,
+            'title': title,
             'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'filesize_approx': filesize_approx,
+            'tbr': bitrate,
             'categories': categories,
-            'age_limit': 18,
+            'age_limit': age_limit,
         }

From 1b7b1d6eac4b40eddd0d574e5e5222f949704104 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Dec 2014 15:41:52 +0600
Subject: [PATCH 308/974] [arte.tv:+7] Make quality optional (Closes #4583)

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 219631b9b..f02539327 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -133,7 +133,7 @@ class ArteTVPlus7IE(InfoExtractor):
                 'width': int_or_none(f.get('width')),
                 'height': int_or_none(f.get('height')),
                 'tbr': int_or_none(f.get('bitrate')),
-                'quality': qfunc(f['quality']),
+                'quality': qfunc(f.get('quality')),
                 'source_preference': source_pref,
             }
 

From 5a1840305746b7242c8d8307e945b9f889603a8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Dec 2014 15:42:29 +0600
Subject: [PATCH 309/974] [arte.tv] Fix typo

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index f02539327..929dd3cc5 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -37,7 +37,7 @@ class ArteTvIE(InfoExtractor):
             config_xml_url, video_id, note='Downloading configuration')
 
         formats = [{
-            'forma_id': q.attrib['quality'],
+            'format_id': q.attrib['quality'],
             # The playpath starts at 'mp4:', if we don't manually
             # split the url, rtmpdump will incorrectly parse them
             'url': q.text.split('mp4:', 1)[0],

From ff7a07d5c4d260a8b27fa7bf523c405725f9cb20 Mon Sep 17 00:00:00 2001
From: Johannes Knoedtel <johannes@dev-baron.de>
Date: Sun, 28 Dec 2014 20:04:21 +0100
Subject: [PATCH 310/974] [archiveorg] most metadata fields are optional

Example: https://archive.org/details/Cops1922
---
 youtube_dl/extractor/archiveorg.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index 34ce8429b..ba94b5454 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -25,6 +25,12 @@ class ArchiveOrgIE(InfoExtractor):
         }
     }
 
+    def get_optional_metadata(self, data, field):
+        try:
+            return data['metadata'][field][0]
+        except KeyError:
+            return None
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -33,10 +39,12 @@ class ArchiveOrgIE(InfoExtractor):
         json_data = self._download_webpage(json_url, video_id)
         data = json.loads(json_data)
 
-        title = data['metadata']['title'][0]
-        description = data['metadata']['description'][0]
-        uploader = data['metadata']['creator'][0]
-        upload_date = unified_strdate(data['metadata']['date'][0])
+        title = self.get_optional_metadata(data, 'title')
+        description = self.get_optional_metadata(data, 'description')
+        uploader = self.get_optional_metadata(data, 'creator')
+        upload_date = self.get_optional_metadata(data, 'date')
+        if upload_date:
+            upload_date = unified_strdate(upload_date)
 
         formats = [
             {

From e8e28989eb8ab0dbb755a273cd1d79847494744b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Dec 2014 02:08:46 +0600
Subject: [PATCH 311/974] [archiveorg] Add test, simplify and modernize

---
 youtube_dl/extractor/archiveorg.py | 60 +++++++++++++++---------------
 1 file changed, 29 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index ba94b5454..9fc35a42b 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -1,50 +1,48 @@
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    unified_strdate,
-)
+from ..utils import unified_strdate
 
 
 class ArchiveOrgIE(InfoExtractor):
     IE_NAME = 'archive.org'
     IE_DESC = 'archive.org videos'
-    _VALID_URL = r'(?:https?://)?(?:www\.)?archive\.org/details/(?P<id>[^?/]+)(?:[?].*)?$'
-    _TEST = {
-        "url": "http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect",
-        'file': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect.ogv',
+    _VALID_URL = r'https?://(?:www\.)?archive\.org/details/(?P<id>[^?/]+)(?:[?].*)?$'
+    _TESTS = [{
+        'url': 'http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
         'md5': '8af1d4cf447933ed3c7f4871162602db',
         'info_dict': {
-            "title": "1968 Demo - FJCC Conference Presentation Reel #1",
-            "description": "Reel 1 of 3: Also known as the \"Mother of All Demos\", Doug Engelbart's presentation at the Fall Joint Computer Conference in San Francisco, December 9, 1968 titled \"A Research Center for Augmenting Human Intellect.\" For this presentation, Doug and his team astonished the audience by not only relating their research, but demonstrating it live. This was the debut of the mouse, interactive computing, hypermedia, computer supported software engineering, video teleconferencing, etc. See also <a href=\"http://dougengelbart.org/firsts/dougs-1968-demo.html\" rel=\"nofollow\">Doug's 1968 Demo page</a> for more background, highlights, links, and the detailed paper published in this conference proceedings. Filmed on 3 reels: Reel 1 | <a href=\"http://www.archive.org/details/XD300-24_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 2</a> | <a href=\"http://www.archive.org/details/XD300-25_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 3</a>",
-            "upload_date": "19681210",
-            "uploader": "SRI International"
+            'id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect',
+            'ext': 'ogv',
+            'title': '1968 Demo - FJCC Conference Presentation Reel #1',
+            'description': 'md5:1780b464abaca9991d8968c877bb53ed',
+            'upload_date': '19681210',
+            'uploader': 'SRI International'
         }
-    }
-
-    def get_optional_metadata(self, data, field):
-        try:
-            return data['metadata'][field][0]
-        except KeyError:
-            return None
+    }, {
+        'url': 'https://archive.org/details/Cops1922',
+        'md5': '18f2a19e6d89af8425671da1cf3d4e04',
+        'info_dict': {
+            'id': 'Cops1922',
+            'ext': 'ogv',
+            'title': 'Buster Keaton\'s "Cops" (1922)',
+            'description': 'md5:70f72ee70882f713d4578725461ffcc3',
+        }
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         json_url = url + ('?' if '?' in url else '&') + 'output=json'
-        json_data = self._download_webpage(json_url, video_id)
-        data = json.loads(json_data)
+        data = self._download_json(json_url, video_id)
 
-        title = self.get_optional_metadata(data, 'title')
-        description = self.get_optional_metadata(data, 'description')
-        uploader = self.get_optional_metadata(data, 'creator')
-        upload_date = self.get_optional_metadata(data, 'date')
-        if upload_date:
-            upload_date = unified_strdate(upload_date)
+        def get_optional(data_dict, field):
+            return data_dict['metadata'].get(field, [None])[0]
+
+        title = get_optional(data, 'title')
+        description = get_optional(data, 'description')
+        uploader = get_optional(data, 'creator')
+        upload_date = unified_strdate(get_optional(data, 'date'))
 
         formats = [
             {

From ae6986fb895ceb72e084b587bc9d070c86848347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Dec 2014 03:00:24 +0600
Subject: [PATCH 312/974] [bbccouk] Switch to new JSON playlist format (Closes
 #4588)

---
 youtube_dl/extractor/bbccouk.py | 95 +++++++++++++++++++++++----------
 1 file changed, 68 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 2d2f742ae..f690dc803 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -71,7 +71,20 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
                 'skip_download': True,
             },
             'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
-        },
+        }, {
+            'url': 'http://www.bbc.co.uk/programmes/b04v20dw',
+            'info_dict': {
+                'id': 'b04v209v',
+                'ext': 'flv',
+                'title': 'Pete Tong, The Essential New Tune Special',
+                'description': "Pete has a very special mix - all of 2014's Essential New Tunes!",
+                'duration': 10800,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            }
+        }
     ]
 
     def _extract_asx_playlist(self, connection, programme_id):
@@ -203,6 +216,59 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
 
         return formats, subtitles
 
+    def _download_playlist(self, playlist_id):
+        try:
+            playlist = self._download_json(
+                'http://www.bbc.co.uk/programmes/%s/playlist.json' % playlist_id,
+                playlist_id, 'Downloading playlist JSON')
+
+            version = playlist.get('defaultAvailableVersion')
+            if version:
+                smp_config = version['smpConfig']
+                title = smp_config['title']
+                description = smp_config['summary']
+                for item in smp_config['items']:
+                    kind = item['kind']
+                    if kind != 'programme' and kind != 'radioProgramme':
+                        continue
+                    programme_id = item.get('vpid')
+                    duration = int(item.get('duration'))
+                    formats, subtitles = self._download_media_selector(programme_id)
+                return programme_id, title, description, duration, formats, subtitles
+        except ExtractorError as ee:
+            if not isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
+                raise
+
+        # fallback to legacy playlist
+        playlist = self._download_xml(
+                'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id,
+                playlist_id, 'Downloading legacy playlist XML')
+
+        no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
+        if no_items is not None:
+            reason = no_items.get('reason')
+            if reason == 'preAvailability':
+                msg = 'Episode %s is not yet available' % playlist_id
+            elif reason == 'postAvailability':
+                msg = 'Episode %s is no longer available' % playlist_id
+            elif reason == 'noMedia':
+                msg = 'Episode %s is not currently available' % playlist_id
+            else:
+                msg = 'Episode %s is not available: %s' % (playlist_id, reason)
+            raise ExtractorError(msg, expected=True)
+
+        for item in self._extract_items(playlist):
+            kind = item.get('kind')
+            if kind != 'programme' and kind != 'radioProgramme':
+                continue
+            title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
+            description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
+            programme_id = item.get('identifier')
+            duration = int(item.get('duration'))
+            formats, subtitles = self._download_media_selector(programme_id)
+
+        return programme_id, title, description, duration, formats, subtitles
+
     def _real_extract(self, url):
         group_id = self._match_id(url)
 
@@ -219,32 +285,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             duration = player['duration']
             formats, subtitles = self._download_media_selector(programme_id)
         else:
-            playlist = self._download_xml(
-                'http://www.bbc.co.uk/iplayer/playlist/%s' % group_id,
-                group_id, 'Downloading playlist XML')
-
-            no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
-            if no_items is not None:
-                reason = no_items.get('reason')
-                if reason == 'preAvailability':
-                    msg = 'Episode %s is not yet available' % group_id
-                elif reason == 'postAvailability':
-                    msg = 'Episode %s is no longer available' % group_id
-                elif reason == 'noMedia':
-                    msg = 'Episode %s is not currently available' % group_id
-                else:
-                    msg = 'Episode %s is not available: %s' % (group_id, reason)
-                raise ExtractorError(msg, expected=True)
-
-            for item in self._extract_items(playlist):
-                kind = item.get('kind')
-                if kind != 'programme' and kind != 'radioProgramme':
-                    continue
-                title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
-                description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
-                programme_id = item.get('identifier')
-                duration = int(item.get('duration'))
-                formats, subtitles = self._download_media_selector(programme_id)
+            programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)
 
         if self._downloader.params.get('listsubtitles', False):
             self._list_available_subtitles(programme_id, subtitles)

From 513fd2a8723b640844d9fe7adaa8e14fe6746ec2 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Mon, 29 Dec 2014 10:38:07 +0100
Subject: [PATCH 313/974] [hellporno] simplify

---
 youtube_dl/extractor/hellporno.py | 32 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/hellporno.py b/youtube_dl/extractor/hellporno.py
index 06f85127f..754322cdf 100644
--- a/youtube_dl/extractor/hellporno.py
+++ b/youtube_dl/extractor/hellporno.py
@@ -1,55 +1,51 @@
 from __future__ import unicode_literals
-
-import re
-
 from .common import InfoExtractor
 
+
 class HellPornoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?hellporno\.com/videos/(?P<display_id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?hellporno\.com/videos/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://hellporno.com/videos/dixie-is-posing-with-naked-ass-very-erotic/',
         'md5': '1fee339c610d2049699ef2aa699439f1',
         'info_dict': {
             'id': '149116',
+            'display_id': 'dixie-is-posing-with-naked-ass-very-erotic',
             'ext': 'mp4',
             'title': 'Dixie is posing with naked ass very erotic',
-            'description': 'md5:5ba02cbf31eff820147b3cc25306d89a',
-            'categories': list,  # NSFW
             'thumbnail': 're:https?://.*\.jpg$',
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        webpage = self._download_webpage(url, 'main')
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
 
-        video_id = self._html_search_regex(r'video_id:\s*\'([^\']+)\'', webpage, 'id')
+        video_id = self._html_search_regex(
+            r'video_id:\s*\'([^\']+)\'', webpage, 'id')
 
-        video_url = self._html_search_regex(r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
+        ext = self._html_search_regex(
+            r'postfix:\s*\'([^\']+)\'', webpage, 'ext')[1:]
 
-        ext = self._html_search_regex(r'postfix:\s*\'([^\']+)\'', webpage, 'ext')[1:]
+        video_url = self._html_search_regex(
+            r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
 
         title = self._html_search_regex(
             r'<title>([^<]+)\s*-\s*Hell Porno</title>', webpage, 'title')
 
-        description = self._html_search_meta('description', webpage, 'description', fatal=False)
-
         thumbnail = self._html_search_regex(
             r'preview_url:\s*\'([^\']+)\'',
             webpage, 'thumbnail', fatal=False)
 
-        categories_str = self._html_search_regex(
-            r'<meta name="keywords" content="([^"]+)"', webpage, 'categories', fatal=False)
-        categories = (
-            None if categories_str is None
-            else categories_str.split(','))
+        categories = self._html_search_meta(
+            'keywords', webpage, 'categories', default='').split(',')
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'url': video_url,
             'title': title,
             'ext': ext,
-            'description': description,
             'thumbnail': thumbnail,
             'categories': categories,
             'age_limit': 18,

From 0cc4f8e3850e41dc5e3f237315a20eac5eb99f46 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Mon, 29 Dec 2014 11:31:22 +0100
Subject: [PATCH 314/974] [xxxymovies] new ectractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/xxxymovies.py | 63 ++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/xxxymovies.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fd0ebffe3..c8a77616e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -512,6 +512,7 @@ from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
+from .xxxymovies import XXXYMoviesIE
 from .yahoo import (
     YahooIE,
     YahooSearchIE,
diff --git a/youtube_dl/extractor/xxxymovies.py b/youtube_dl/extractor/xxxymovies.py
new file mode 100644
index 000000000..4a9144661
--- /dev/null
+++ b/youtube_dl/extractor/xxxymovies.py
@@ -0,0 +1,63 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    int_or_none,
+)
+
+
+class XXXYMoviesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?xxxymovies\.com/videos/(?P<id>\d+)/(?P<display_id>[^/]+)'
+    _TEST = {
+        'url': 'http://xxxymovies.com/videos/138669/ecstatic-orgasm-sofcore/',
+        'md5': '810b1bdbbffff89dd13bdb369fe7be4b',
+        'info_dict': {
+            'id': '138669',
+            'display_id': 'ecstatic-orgasm-sofcore',
+            'ext': 'mp4',
+            'title': 'Ecstatic Orgasm Sofcore',
+            'duration': 931,
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._html_search_regex(
+            r"video_url\s*:\s*'([^']+)'", webpage, 'video URL')
+
+        title = self._html_search_regex(
+            r'<title>(.*?)\s*-\s*XXXYMovies.com</title>', webpage, 'title')
+
+        thumbnail = self._html_search_regex(
+            r'preview_url\s*:\s*\'(.*?)\'', webpage, 'thumbnail', fatal=False)
+
+        categories = self._html_search_meta(
+            'keywords', webpage, 'categories', default='').split(',')
+
+        duration = parse_duration(self._search_regex(
+            r'<span>Duration:</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))
+
+        view_count = int_or_none(self._html_search_regex(
+            r'<div class="video_views">\s*(\d+)', webpage, 'view count', fatal=False))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'duration': duration,
+            'view_count': view_count,
+            'age_limit': 18,
+        }

From 7fc2cd819e10ca6c69cb3601c0bbe7d577256da7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Dec 2014 20:27:09 +0600
Subject: [PATCH 315/974] [cnn] Improve regexes and fix test

---
 youtube_dl/extractor/cnn.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index cddcacdf1..1bff005d6 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -11,14 +11,14 @@ from ..utils import (
 
 
 class CNNIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://((edition|www)\.)?cnn\.com/video/(data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn(-ap)?|(?=&)))'''
+    _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn(?:-ap)?|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
         'md5': '3e6121ea48df7e2259fe73a0628605c4',
         'info_dict': {
-            'id': 'sports_2013_06_09_nadal-1-on-1.cnn',
+            'id': 'sports/2013/06/09/nadal-1-on-1.cnn',
             'ext': 'mp4',
             'title': 'Nadal wins 8th French Open title',
             'description': 'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.',
@@ -130,7 +130,7 @@ class CNNBlogsIE(InfoExtractor):
 
 
 class CNNArticleIE(InfoExtractor):
-    _VALID_URL = r'https?://((edition|www)\.)?cnn\.com/(?!video/)'
+    _VALID_URL = r'https?://(?:(?:edition|www)\.)?cnn\.com/(?!video/)'
     _TEST = {
         'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
         'md5': '275b326f85d80dff7592a9820f5dc887',
@@ -149,6 +149,6 @@ class CNNArticleIE(InfoExtractor):
         cnn_url = self._html_search_regex(r"video:\s*'([^']+)'", webpage, 'cnn url')
         return {
             '_type': 'url',
-            'url': r'http://cnn.com/video/?/video/' + cnn_url,
+            'url': 'http://cnn.com/video/?/video/' + cnn_url,
             'ie_key': CNNIE.ie_key(),
         }

From 6343a5f68e6dbf5a4fc1b0c7d0bec9e1ddc57143 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Dec 2014 21:05:21 +0600
Subject: [PATCH 316/974] [xxxymovies] Improve

---
 youtube_dl/extractor/xxxymovies.py | 36 ++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/xxxymovies.py b/youtube_dl/extractor/xxxymovies.py
index 4a9144661..5c8f17eb2 100644
--- a/youtube_dl/extractor/xxxymovies.py
+++ b/youtube_dl/extractor/xxxymovies.py
@@ -20,35 +20,51 @@ class XXXYMoviesIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Ecstatic Orgasm Sofcore',
             'duration': 931,
+            'categories': list,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-
         video_id = mobj.group('id')
         display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id)
 
-        video_url = self._html_search_regex(
+        video_url = self._search_regex(
             r"video_url\s*:\s*'([^']+)'", webpage, 'video URL')
 
         title = self._html_search_regex(
-            r'<title>(.*?)\s*-\s*XXXYMovies.com</title>', webpage, 'title')
+            [r'<div class="block_header">\s*<h1>([^<]+)</h1>',
+             r'<title>(.*?)\s*-\s*XXXYMovies\.com</title>'],
+            webpage, 'title')
 
-        thumbnail = self._html_search_regex(
-            r'preview_url\s*:\s*\'(.*?)\'', webpage, 'thumbnail', fatal=False)
+        thumbnail = self._search_regex(
+            r"preview_url\s*:\s*'([^']+)'",
+            webpage, 'thumbnail', fatal=False)
 
         categories = self._html_search_meta(
             'keywords', webpage, 'categories', default='').split(',')
 
         duration = parse_duration(self._search_regex(
-            r'<span>Duration:</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))
+            r'<span>Duration:</span>\s*(\d+:\d+)',
+            webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._html_search_regex(
-            r'<div class="video_views">\s*(\d+)', webpage, 'view count', fatal=False))
+            r'<div class="video_views">\s*(\d+)',
+            webpage, 'view count', fatal=False))
+        like_count = int_or_none(self._search_regex(
+            r'>\s*Likes? <b>\((\d+)\)',
+            webpage, 'like count', fatal=False))
+        dislike_count = int_or_none(self._search_regex(
+            r'>\s*Dislike <b>\((\d+)\)</b>',
+            webpage, 'dislike count', fatal=False))
+
+        age_limit = self._rta_search(webpage)
 
         return {
             'id': video_id,
@@ -59,5 +75,7 @@ class XXXYMoviesIE(InfoExtractor):
             'categories': categories,
             'duration': duration,
             'view_count': view_count,
-            'age_limit': 18,
+            'like_count': like_count,
+            'dislike_count': dislike_count,
+            'age_limit': age_limit,
         }

From 355e41466d8edcb5b2457dfa0c5715ef87ff5832 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Dec 2014 21:33:41 +0600
Subject: [PATCH 317/974] [hellporno] Extract all formats and improve

---
 youtube_dl/extractor/hellporno.py | 47 ++++++++++++++++++++++---------
 1 file changed, 33 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/hellporno.py b/youtube_dl/extractor/hellporno.py
index 754322cdf..7a1c75b65 100644
--- a/youtube_dl/extractor/hellporno.py
+++ b/youtube_dl/extractor/hellporno.py
@@ -1,5 +1,12 @@
 from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    remove_end,
+)
 
 
 class HellPornoIE(InfoExtractor):
@@ -19,23 +26,36 @@ class HellPornoIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+
         webpage = self._download_webpage(url, display_id)
 
-        video_id = self._html_search_regex(
-            r'video_id:\s*\'([^\']+)\'', webpage, 'id')
+        title = remove_end(self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title'), ' - Hell Porno')
 
-        ext = self._html_search_regex(
-            r'postfix:\s*\'([^\']+)\'', webpage, 'ext')[1:]
+        flashvars = self._parse_json(self._search_regex(
+            r'var\s+flashvars\s*=\s*({.+?});', webpage, 'flashvars'),
+            display_id, transform_source=js_to_json)
 
-        video_url = self._html_search_regex(
-            r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
+        video_id = flashvars.get('video_id')
+        thumbnail = flashvars.get('preview_url')
+        ext = flashvars.get('postfix', '.mp4')[1:]
 
-        title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*Hell Porno</title>', webpage, 'title')
-
-        thumbnail = self._html_search_regex(
-            r'preview_url:\s*\'([^\']+)\'',
-            webpage, 'thumbnail', fatal=False)
+        formats = []
+        for video_url_key in ['video_url', 'video_alt_url']:
+            video_url = flashvars.get(video_url_key)
+            if not video_url:
+                continue
+            video_text = flashvars.get('%s_text' % video_url_key)
+            fmt = {
+                'url': video_url,
+                'ext': ext,
+                'format_id': video_text,
+            }
+            m = re.search(r'^(?P<height>\d+)[pP]', video_text)
+            if m:
+                fmt['height'] = int(m.group('height'))
+            formats.append(fmt)
+        self._sort_formats(formats)
 
         categories = self._html_search_meta(
             'keywords', webpage, 'categories', default='').split(',')
@@ -43,10 +63,9 @@ class HellPornoIE(InfoExtractor):
         return {
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
             'title': title,
-            'ext': ext,
             'thumbnail': thumbnail,
             'categories': categories,
             'age_limit': 18,
+            'formats': formats,
         }

From 429ddfd38dfb53aa05f0e2a77aeeb4ac5d20a17d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 01:50:28 +0600
Subject: [PATCH 318/974] [cnn] Add support for hln URL format (Closes #4595)

---
 youtube_dl/extractor/cnn.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 1bff005d6..93e8d0de3 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class CNNIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn(?:-ap)?|(?=&)))'''
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:cnn|hln)(?:-ap)?|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
@@ -35,6 +35,16 @@ class CNNIE(InfoExtractor):
             "description": "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
             "upload_date": "20130821",
         }
+    }, {
+        'url': 'http://www.cnn.com/video/data/2.0/video/living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln.html',
+        'md5': 'f14d02ebd264df951feb2400e2c25a1b',
+        'info_dict': {
+            'id': 'living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln',
+            'ext': 'mp4',
+            'title': 'Nashville Ep. 1: Hand crafted skateboards',
+            'description': 'md5:e7223a503315c9f150acac52e76de086',
+            'upload_date': '20141222',
+        }
     }]
 
     def _real_extract(self, url):

From da3f7fb7f84e47de6aa0b29b16f78cb5bdf7d746 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Sun, 28 Dec 2014 17:07:32 +0000
Subject: [PATCH 319/974] [hitbox] add extractor for hitbox vods

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/hitbox.py   | 104 +++++++++++++++++++++++++++++++
 2 files changed, 105 insertions(+)
 create mode 100644 youtube_dl/extractor/hitbox.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ea4faf2a6..3300dfeb4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -172,6 +172,7 @@ from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
+from .hitbox import HitboxIE
 from .hornbunny import HornBunnyIE
 from .hostingbulk import HostingBulkIE
 from .hotnewhiphop import HotNewHipHopIE
diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
new file mode 100644
index 000000000..239da3cd4
--- /dev/null
+++ b/youtube_dl/extractor/hitbox.py
@@ -0,0 +1,104 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+)
+
+
+class HitboxIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/video/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.hitbox.tv/video/358062',
+        'info_dict': {
+            'id': '358062',
+            'title': 'Megaman',
+            'alt_title': 'Megaman',
+            'description': '',
+            'ext': 'mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 3834,
+            'resolution': 'SD 480p',
+            'uploader_id': 'supergreatfriend',
+            'view_count': int,
+            'upload_date': '20141225',
+            'categories': [None],
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.hitbox.tv/video/203213',
+        'info_dict': {
+            'id': '203213',
+            'title': 'hitbox @ gamescom, Sub Button Hype extended, Giveaway - hitbox News Update with Oxy',
+            'alt_title': 'hitboxlive - Aug 9th #6',
+            'description': '',
+            'ext': 'mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 215,
+            'resolution': 'HD 720p',
+            'uploader_id': 'hitboxlive',
+            'view_count': int,
+            'upload_date': '20140809',
+            'categories': ['Live Show'],
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        thumb_base = 'https://edge.sf.hitbox.tv'
+        metadata = self._download_json(
+            'https://www.hitbox.tv/api/media/video/%s' % (video_id), video_id
+        )
+
+        video_meta = metadata.get('video', [])[0]
+        title = video_meta.get('media_status')
+        alt_title = video_meta.get('media_title')
+        description = video_meta.get('media_description')
+        duration = int(float(video_meta.get('media_duration')))
+        uploader = video_meta.get('media_user_name')
+        views = int(video_meta.get('media_views'))
+        upload_date = unified_strdate(video_meta.get('media_date_added'))
+        categories = [video_meta.get('category_name')]
+        thumbs = [
+            {'url': thumb_base + video_meta.get('media_thumbnail'),
+             'width': 320,
+             'height': 180},
+            {'url': thumb_base + video_meta.get('media_thumbnail_large'),
+             'width': 768,
+             'height': 432},
+        ]
+
+        player_config = self._download_json(
+            'https://www.hitbox.tv/api/player/config/video/%s' % (video_id),
+            video_id
+        )
+
+        clip = player_config.get('clip')
+        video_url = clip.get('url')
+        res = clip.get('bitrates', [])[0].get('label')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'alt_title': alt_title,
+            'description': description,
+            'url': video_url,
+            'ext': 'mp4',
+            'thumbnails': thumbs,
+            'duration': duration,
+            'resolution': res,
+            'uploader_id': uploader,
+            'view_count': views,
+            'upload_date': upload_date,
+            'categories': categories,
+            'protocol': 'm3u8',
+        }

From e3947e2b7fe1cd81f841daa1c4dc2ca72af8aefe Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Mon, 29 Dec 2014 20:10:59 +0000
Subject: [PATCH 320/974] [hitbox] add support for live streams

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/hitbox.py   | 146 +++++++++++++++++++++----------
 2 files changed, 102 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3300dfeb4..4f5a1ce18 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -172,7 +172,7 @@ from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
-from .hitbox import HitboxIE
+from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hostingbulk import HostingBulkIE
 from .hotnewhiphop import HotNewHipHopIE
diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index 239da3cd4..eab2749ec 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -1,5 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
+import re
 
 from .common import InfoExtractor
 from ..utils import (
@@ -9,33 +10,13 @@ from ..utils import (
 
 class HitboxIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/video/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.hitbox.tv/video/358062',
-        'info_dict': {
-            'id': '358062',
-            'title': 'Megaman',
-            'alt_title': 'Megaman',
-            'description': '',
-            'ext': 'mp4',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 3834,
-            'resolution': 'SD 480p',
-            'uploader_id': 'supergreatfriend',
-            'view_count': int,
-            'upload_date': '20141225',
-            'categories': [None],
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
+    _TEST = {
         'url': 'http://www.hitbox.tv/video/203213',
         'info_dict': {
             'id': '203213',
             'title': 'hitbox @ gamescom, Sub Button Hype extended, Giveaway - hitbox News Update with Oxy',
             'alt_title': 'hitboxlive - Aug 9th #6',
-            'description': '',
+            'description': '\n',
             'ext': 'mp4',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 215,
@@ -49,24 +30,28 @@ class HitboxIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
+    }
 
+    def _extract_metadata(self, url, video_id):
         thumb_base = 'https://edge.sf.hitbox.tv'
         metadata = self._download_json(
-            'https://www.hitbox.tv/api/media/video/%s' % (video_id), video_id
+            '%s/%s' % (url, video_id), video_id
         )
 
-        video_meta = metadata.get('video', [])[0]
+        date = 'media_live_since'
+        media_type = 'livestream'
+        if metadata.get('media_type') == 'video':
+            media_type = 'video'
+            date = 'media_date_added'
+
+        video_meta = metadata.get(media_type, [])[0]
         title = video_meta.get('media_status')
         alt_title = video_meta.get('media_title')
-        description = video_meta.get('media_description')
+        description = video_meta.get('media_description_md')
         duration = int(float(video_meta.get('media_duration')))
         uploader = video_meta.get('media_user_name')
         views = int(video_meta.get('media_views'))
-        upload_date = unified_strdate(video_meta.get('media_date_added'))
+        upload_date = unified_strdate(video_meta.get(date))
         categories = [video_meta.get('category_name')]
         thumbs = [
             {'url': thumb_base + video_meta.get('media_thumbnail'),
@@ -77,6 +62,28 @@ class HitboxIE(InfoExtractor):
              'height': 432},
         ]
 
+        return {
+            'id': video_id,
+            'title': title,
+            'alt_title': alt_title,
+            'description': description,
+            'ext': 'mp4',
+            'thumbnails': thumbs,
+            'duration': duration,
+            'uploader_id': uploader,
+            'view_count': views,
+            'upload_date': upload_date,
+            'categories': categories,
+        }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        metadata = self._extract_metadata(
+            'https://www.hitbox.tv/api/media/video',
+            video_id
+        )
+
         player_config = self._download_json(
             'https://www.hitbox.tv/api/player/config/video/%s' % (video_id),
             video_id
@@ -86,19 +93,68 @@ class HitboxIE(InfoExtractor):
         video_url = clip.get('url')
         res = clip.get('bitrates', [])[0].get('label')
 
-        return {
-            'id': video_id,
-            'title': title,
-            'alt_title': alt_title,
-            'description': description,
-            'url': video_url,
+        metadata['resolution'] = res
+        metadata['url'] = video_url
+        metadata['protocol'] = 'm3u8'
+
+        return metadata
+
+
+class HitboxLiveIE(HitboxIE):
+    _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/(?!video)(?P<id>.+)'
+    _TEST = {
+        'url': 'http://www.hitbox.tv/dimak',
+        'info_dict': {
+            'id': 'dimak',
             'ext': 'mp4',
-            'thumbnails': thumbs,
-            'duration': duration,
-            'resolution': res,
-            'uploader_id': uploader,
-            'view_count': views,
-            'upload_date': upload_date,
-            'categories': categories,
-            'protocol': 'm3u8',
-        }
+            'description': str,
+            'upload_date': str,
+            'title': str,
+            'uploader_id': 'Dimak',
+        },
+        'params': {
+            # live
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        metadata = self._extract_metadata(
+            'https://www.hitbox.tv/api/media/live',
+            video_id
+        )
+
+        player_config = self._download_json(
+            'https://www.hitbox.tv/api/player/config/live/%s' % (video_id),
+            video_id
+        )
+
+        formats = []
+        cdns = player_config.get('cdns')
+        servers = []
+        for cdn in cdns:
+            base_url = cdn.get('netConnectionUrl')
+            host = re.search('.+\.([^\.]+\.[^\./]+)/.+', base_url).group(1)
+            if base_url not in servers:
+                servers.append(base_url)
+                for stream in cdn.get('bitrates'):
+                    label = stream.get('label')
+                    if label != 'Auto':
+                        formats.append({
+                            'url': '%s/%s' % (base_url, stream.get('url')),
+                            'ext': 'mp4',
+                            'vbr': stream.get('bitrate'),
+                            'resolution': label,
+                            'rtmp_live': True,
+                            'format_note': host,
+                            'page_url': url,
+                            'player_url': 'http://www.hitbox.tv/static/player/flowplayer/flowplayer.commercial-3.2.16.swf',
+                        })
+
+        self._sort_formats(formats)
+        metadata['formats'] = formats
+        metadata['is_live'] = True
+        metadata['title'] = self._live_title(metadata.get('title'))
+        return metadata

From 0c0a70f4c6839903b326d7d9074e93235defaa5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 03:22:07 +0600
Subject: [PATCH 321/974] [hitbox] Minor changes

---
 youtube_dl/extractor/hitbox.py | 58 +++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index eab2749ec..84bd7c080 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -1,14 +1,20 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    unified_strdate,
+    clean_html,
+    parse_iso8601,
+    float_or_none,
+    int_or_none,
+    compat_str,
 )
 
 
 class HitboxIE(InfoExtractor):
+    IE_NAME = 'hitbox'
     _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.hitbox.tv/video/203213',
@@ -16,13 +22,14 @@ class HitboxIE(InfoExtractor):
             'id': '203213',
             'title': 'hitbox @ gamescom, Sub Button Hype extended, Giveaway - hitbox News Update with Oxy',
             'alt_title': 'hitboxlive - Aug 9th #6',
-            'description': '\n',
+            'description': '',
             'ext': 'mp4',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 215,
+            'duration': 215.1666,
             'resolution': 'HD 720p',
-            'uploader_id': 'hitboxlive',
+            'uploader': 'hitboxlive',
             'view_count': int,
+            'timestamp': 1407576133,
             'upload_date': '20140809',
             'categories': ['Live Show'],
         },
@@ -35,8 +42,7 @@ class HitboxIE(InfoExtractor):
     def _extract_metadata(self, url, video_id):
         thumb_base = 'https://edge.sf.hitbox.tv'
         metadata = self._download_json(
-            '%s/%s' % (url, video_id), video_id
-        )
+            '%s/%s' % (url, video_id), video_id)
 
         date = 'media_live_since'
         media_type = 'livestream'
@@ -47,11 +53,13 @@ class HitboxIE(InfoExtractor):
         video_meta = metadata.get(media_type, [])[0]
         title = video_meta.get('media_status')
         alt_title = video_meta.get('media_title')
-        description = video_meta.get('media_description_md')
-        duration = int(float(video_meta.get('media_duration')))
+        description = clean_html(
+            video_meta.get('media_description') or
+            video_meta.get('media_description_md'))
+        duration = float_or_none(video_meta.get('media_duration'))
         uploader = video_meta.get('media_user_name')
-        views = int(video_meta.get('media_views'))
-        upload_date = unified_strdate(video_meta.get(date))
+        views = int_or_none(video_meta.get('media_views'))
+        timestamp = parse_iso8601(video_meta.get(date), ' ')
         categories = [video_meta.get('category_name')]
         thumbs = [
             {'url': thumb_base + video_meta.get('media_thumbnail'),
@@ -70,9 +78,9 @@ class HitboxIE(InfoExtractor):
             'ext': 'mp4',
             'thumbnails': thumbs,
             'duration': duration,
-            'uploader_id': uploader,
+            'uploader': uploader,
             'view_count': views,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'categories': categories,
         }
 
@@ -81,13 +89,11 @@ class HitboxIE(InfoExtractor):
 
         metadata = self._extract_metadata(
             'https://www.hitbox.tv/api/media/video',
-            video_id
-        )
+            video_id)
 
         player_config = self._download_json(
-            'https://www.hitbox.tv/api/player/config/video/%s' % (video_id),
-            video_id
-        )
+            'https://www.hitbox.tv/api/player/config/video/%s' % video_id,
+            video_id)
 
         clip = player_config.get('clip')
         video_url = clip.get('url')
@@ -101,16 +107,18 @@ class HitboxIE(InfoExtractor):
 
 
 class HitboxLiveIE(HitboxIE):
+    IE_NAME = 'hitbox:live'
     _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/(?!video)(?P<id>.+)'
     _TEST = {
         'url': 'http://www.hitbox.tv/dimak',
         'info_dict': {
             'id': 'dimak',
             'ext': 'mp4',
-            'description': str,
-            'upload_date': str,
-            'title': str,
-            'uploader_id': 'Dimak',
+            'description': 'md5:c9f80fa4410bc588d7faa40003fc7d0e',
+            'timestamp': int,
+            'upload_date': compat_str,
+            'title': compat_str,
+            'uploader': 'Dimak',
         },
         'params': {
             # live
@@ -123,13 +131,11 @@ class HitboxLiveIE(HitboxIE):
 
         metadata = self._extract_metadata(
             'https://www.hitbox.tv/api/media/live',
-            video_id
-        )
+            video_id)
 
         player_config = self._download_json(
-            'https://www.hitbox.tv/api/player/config/live/%s' % (video_id),
-            video_id
-        )
+            'https://www.hitbox.tv/api/player/config/live/%s' % video_id,
+            video_id)
 
         formats = []
         cdns = player_config.get('cdns')

From beb95e778170895c1435f55b71e3d76cbd9d8bcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ce=CC=81dric=20Luthi?= <cedric.luthi@gmail.com>
Date: Mon, 29 Dec 2014 22:58:14 +0100
Subject: [PATCH 322/974] [youtube] Fix videos with age gate and encrypted
 signatures

The `sts` value is available on the embed webpage, get it from there.

Fixes #4108.
---
 youtube_dl/extractor/youtube.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 550e18733..175e43272 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -766,11 +766,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
             # this can be viewed without login into Youtube
+            url = proto + '://www.youtube.com/embed/%s' % video_id
+            embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
             data = compat_urllib_parse.urlencode({
                 'video_id': video_id,
                 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
                 'sts': self._search_regex(
-                    r'"sts"\s*:\s*(\d+)', video_webpage, 'sts', default=''),
+                    r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
             })
             video_info_url = proto + '://www.youtube.com/get_video_info?' + data
             video_info_webpage = self._download_webpage(
@@ -968,11 +970,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 elif 's' in url_data:
                     encrypted_sig = url_data['s'][0]
 
-                    if not age_gate:
-                        jsplayer_url_json = self._search_regex(
-                            r'"assets":.+?"js":\s*("[^"]+")',
-                            video_webpage, 'JS player URL')
-                        player_url = json.loads(jsplayer_url_json)
+                    jsplayer_url_json = self._search_regex(
+                        r'"assets":.+?"js":\s*("[^"]+")',
+                        embed_webpage if age_gate else video_webpage, 'JS player URL')
+                    player_url = json.loads(jsplayer_url_json)
                     if player_url is None:
                         player_url_json = self._search_regex(
                             r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',

From a87bb090d95bdfe02429c37a2a10a4733ae69fc6 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Mon, 29 Dec 2014 23:06:56 +0000
Subject: [PATCH 323/974] [daum] update 'full id' regex

fixes #4566
---
 youtube_dl/extractor/daum.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index c6b813f58..b4b3b795c 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -38,7 +38,7 @@ class DaumIE(InfoExtractor):
         canonical_url = 'http://tvpot.daum.net/v/%s' % video_id
         webpage = self._download_webpage(canonical_url, video_id)
         full_id = self._search_regex(
-            r'<iframe src="http://videofarm.daum.net/controller/video/viewer/Video.html\?.*?vid=(.+?)[&"]',
+            r'src=["\']http://videofarm.daum.net/controller/video/viewer/Video.html\?.*?vid=(.+?)[&"\']',
             webpage, 'full id')
         query = compat_urllib_parse.urlencode({'vid': full_id})
         info = self._download_xml(

From 1c57e7f1f496a6f1fad43c0b193ce534fc9ce67b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 16:55:53 +0600
Subject: [PATCH 324/974] [daum] Improve full_id regex

---
 youtube_dl/extractor/daum.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index b4b3b795c..934da765e 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -38,7 +38,7 @@ class DaumIE(InfoExtractor):
         canonical_url = 'http://tvpot.daum.net/v/%s' % video_id
         webpage = self._download_webpage(canonical_url, video_id)
         full_id = self._search_regex(
-            r'src=["\']http://videofarm.daum.net/controller/video/viewer/Video.html\?.*?vid=(.+?)[&"\']',
+            r'src=["\']http://videofarm\.daum\.net/controller/video/viewer/Video\.html\?.*?vid=(.+?)[&"\']',
             webpage, 'full id')
         query = compat_urllib_parse.urlencode({'vid': full_id})
         info = self._download_xml(

From 3ee08848dbc2314df39e6931c75920b406733868 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 17:12:12 +0600
Subject: [PATCH 325/974] Credit @0xced for #4598

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bb4d8b4d1..29ce9e3e4 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -96,3 +96,4 @@ Mathias Rav
 Petr Kutalek
 Will Glynn
 Max Reimann
+Cédric Luthi

From fccae2b911970d0ffa97800b27e70b1937cd3058 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 17:26:21 +0600
Subject: [PATCH 326/974] [youtube] Add test for age-gate video with encrypted
 signature

---
 youtube_dl/extractor/youtube.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 175e43272..3da83e3a8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -418,6 +418,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'upload_date': '20140605',
             },
         },
+        # Age-gate video with encrypted signature
+        {
+            'url': 'http://www.youtube.com/watch?v=6kLq3WMV1nU',
+            'info_dict': {
+                'id': '6kLq3WMV1nU',
+                'ext': 'mp4',
+                'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
+                'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
+                'uploader': 'LloydVEVO',
+                'uploader_id': 'LloydVEVO',
+                'upload_date': '20110629',
+            },
+        },
         # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
         {
             'url': '__2ABJjxzNo',

From a349873226c873a2f3bea9f5dffe167053cb1666 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 22:28:07 +0600
Subject: [PATCH 327/974] [atresplayer] Add extractor (Closes #2341)

---
 youtube_dl/extractor/__init__.py    |   1 +
 youtube_dl/extractor/atresplayer.py | 111 ++++++++++++++++++++++++++++
 2 files changed, 112 insertions(+)
 create mode 100644 youtube_dl/extractor/atresplayer.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4f5a1ce18..c15786ad7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -25,6 +25,7 @@ from .arte import (
     ArteTVDDCIE,
     ArteTVEmbedIE,
 )
+from .atresplayer import AtresPlayerIE
 from .audiomack import AudiomackIE
 from .auengine import AUEngineIE
 from .azubu import AzubuIE
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
new file mode 100644
index 000000000..7e987b2a0
--- /dev/null
+++ b/youtube_dl/extractor/atresplayer.py
@@ -0,0 +1,111 @@
+from __future__ import unicode_literals
+
+import time
+import hmac
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_str,
+    compat_urllib_request,
+    int_or_none,
+    float_or_none,
+    xpath_text,
+    ExtractorError,
+)
+
+
+class AtresPlayerIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'
+    _TESTS = [
+        {
+            'url': 'http://www.atresplayer.com/television/programas/el-club-de-la-comedia/temporada-4/capitulo-10-especial-solidario-nochebuena_2014122100174.html',
+            'md5': 'efd56753cda1bb64df52a3074f62e38a',
+            'info_dict': {
+                'id': 'capitulo-10-especial-solidario-nochebuena',
+                'ext': 'mp4',
+                'title': 'Especial Solidario de Nochebuena',
+                'description': 'md5:e2d52ff12214fa937107d21064075bf1',
+                'duration': 5527.6,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+        },
+        {
+            'url': 'http://www.atresplayer.com/television/series/el-secreto-de-puente-viejo/el-chico-de-los-tres-lunares/capitulo-977-29-12-14_2014122400174.html',
+            'only_matching': True,
+        },
+    ]
+
+    _USER_AGENT = 'Dalvik/1.6.0 (Linux; U; Android 4.3; GT-I9300 Build/JSS15J'
+    _MAGIC = 'QWtMLXs414Yo+c#_+Q#K@NN)'
+    _TIMESTAMP_SHIFT = 30000
+
+    _TIME_API_URL = 'http://servicios.atresplayer.com/api/admin/time.json'
+    _URL_VIDEO_TEMPLATE = 'https://servicios.atresplayer.com/api/urlVideo/{1}/{0}/{1}|{2}|{3}.json'
+    _PLAYER_URL_TEMPLATE = 'https://servicios.atresplayer.com/episode/getplayer.json?episodePk=%s'
+    _EPISODE_URL_TEMPLATE = 'http://www.atresplayer.com/episodexml/%s'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        episode_id = self._search_regex(
+            r'episode="([^"]+)"', webpage, 'episode id')
+
+        timestamp = int_or_none(self._download_webpage(
+            self._TIME_API_URL,
+            video_id, 'Downloading timestamp', fatal=False), 1000, time.time())
+        timestamp_shifted = compat_str(timestamp + self._TIMESTAMP_SHIFT)
+        token = hmac.new(self._MAGIC.encode('utf-8'), episode_id + timestamp_shifted).hexdigest()
+
+        formats = []
+        for fmt in ['windows', 'android_tablet']:
+            request = compat_urllib_request.Request(
+                self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
+            request.add_header('Youtubedl-user-agent', self._USER_AGENT)
+
+            fmt_json = self._download_json(
+                request, video_id, 'Downloading %s video JSON' % fmt)
+
+            result = fmt_json.get('resultDes')
+            if result.lower() != 'ok':
+                raise ExtractorError(
+                    '%s returned error: %s' % (self.IE_NAME, result), expected=True)
+
+            for _, video_url in fmt_json['resultObject'].items():
+                if video_url.endswith('/Manifest'):
+                    formats.extend(self._extract_f4m_formats(video_url[:-9] + '/manifest.f4m', video_id))
+                else:
+                    formats.append({
+                        'url': video_url,
+                        'format_id': 'android',
+                        'preference': 1,
+                    })
+        self._sort_formats(formats)
+
+        player = self._download_json(
+            self._PLAYER_URL_TEMPLATE % episode_id,
+            episode_id)
+
+        path_data = player.get('pathData')
+
+        episode = self._download_xml(
+            self._EPISODE_URL_TEMPLATE % path_data,
+            video_id, 'Downloading episode XML')
+
+        duration = float_or_none(xpath_text(
+            episode, './media/asset/info/technical/contentDuration', 'duration'))
+
+        art = episode.find('./media/asset/info/art')
+        title = xpath_text(art, './name', 'title')
+        description = xpath_text(art, './description', 'description')
+        thumbnail = xpath_text(episode, './media/asset/files/background', 'thumbnail')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }

From e83eebb12f984c1614204e53c09dc5124b52b45c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 22:46:23 +0600
Subject: [PATCH 328/974] [atresplayer] Fix python3 bug

---
 youtube_dl/extractor/atresplayer.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 7e987b2a0..72e83bfc2 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -56,7 +56,10 @@ class AtresPlayerIE(InfoExtractor):
             self._TIME_API_URL,
             video_id, 'Downloading timestamp', fatal=False), 1000, time.time())
         timestamp_shifted = compat_str(timestamp + self._TIMESTAMP_SHIFT)
-        token = hmac.new(self._MAGIC.encode('utf-8'), episode_id + timestamp_shifted).hexdigest()
+        token = hmac.new(
+            self._MAGIC.encode('ascii'),
+            (episode_id + timestamp_shifted).encode('utf-8')
+        ).hexdigest()
 
         formats = []
         for fmt in ['windows', 'android_tablet']:

From 416c7fcbce86324587afae11414c71ff603ad296 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 30 Dec 2014 19:35:35 +0100
Subject: [PATCH 329/974] Add documentation about supported sites (Fixes #4503)

---
 Makefile                          |   7 +-
 README.md                         |   8 +
 devscripts/make_supportedsites.py |  45 +++
 docs/supportedsites.md            | 500 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/generic.py   |   3 +-
 youtube_dl/utils.py               |   7 +
 6 files changed, 567 insertions(+), 3 deletions(-)
 create mode 100644 devscripts/make_supportedsites.py
 create mode 100644 docs/supportedsites.md

diff --git a/Makefile b/Makefile
index 71c8553d8..71470eedb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
+all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json CONTRIBUTING.md.tmp
@@ -50,7 +50,7 @@ offlinetest: codetest
 
 tar: youtube-dl.tar.gz
 
-.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest
+.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest supportedsites
 
 pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish
 
@@ -68,6 +68,9 @@ README.md: youtube_dl/*.py youtube_dl/*/*.py
 CONTRIBUTING.md: README.md
 	python devscripts/make_contributing.py README.md CONTRIBUTING.md
 
+supportedsites:
+	python devscripts/make_supportedsites.py docs/supportedsites.md
+
 README.txt: README.md
 	pandoc -f markdown -t plain README.md -o README.txt
 
diff --git a/README.md b/README.md
index b4afe17d2..93a2750c2 100644
--- a/README.md
+++ b/README.md
@@ -449,6 +449,14 @@ Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unz
 
 To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistributable Package](http://www.microsoft.com/en-us/download/details.aspx?id=29).
 
+### How can I detect whether a given URL is supported by youtube-dl?
+
+For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+
+It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which maches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
+
+If you want to find out, simply call youtube-dl. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
+
 # DEVELOPER INSTRUCTIONS
 
 Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py
new file mode 100644
index 000000000..140010644
--- /dev/null
+++ b/devscripts/make_supportedsites.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import io
+import optparse
+import os
+import sys
+
+
+# Import youtube_dl
+ROOT_DIR = os.path.join(os.path.dirname(__file__), '..')
+sys.path.append(ROOT_DIR)
+import youtube_dl
+
+
+def main():
+    parser = optparse.OptionParser(usage='%prog OUTFILE.md')
+    options, args = parser.parse_args()
+    if len(args) != 1:
+        parser.error('Expected an output filename')
+
+    outfile, = args
+
+    def gen_ies_md(ies):
+        for ie in ies:
+            ie_md = '**{}**'.format(ie.IE_NAME)
+            ie_desc = getattr(ie, 'IE_DESC', None)
+            if ie_desc is False:
+                continue
+            if ie_desc is not None:
+                ie_md += ': {}'.format(ie.IE_DESC)
+            if not ie.working():
+                ie_md += ' (Currently broken)'
+            yield ie_md
+
+    ies = sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower())
+    out = '# Supported sites\n' + ''.join(
+        ' - ' + md + '\n'
+        for md in gen_ies_md(ies))
+
+    with io.open(outfile, 'w', encoding='utf-8') as outf:
+        outf.write(out)
+
+if __name__ == '__main__':
+    main()
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
new file mode 100644
index 000000000..dbbf4a797
--- /dev/null
+++ b/docs/supportedsites.md
@@ -0,0 +1,500 @@
+# Supported sites
+ - **1up.com**
+ - **220.ro**
+ - **24video**
+ - **3sat**
+ - **4tube**
+ - **56.com**
+ - **5min**
+ - **8tracks**
+ - **9gag**
+ - **abc.net.au**
+ - **AcademicEarth:Course**
+ - **AddAnime**
+ - **AdobeTV**
+ - **AdultSwim**
+ - **Aftonbladet**
+ - **AlJazeera**
+ - **Allocine**
+ - **anitube.se**
+ - **AnySex**
+ - **Aparat**
+ - **AppleTrailers**
+ - **archive.org**: archive.org videos
+ - **ARD**
+ - **ARD:mediathek**
+ - **arte.tv**
+ - **arte.tv:+7**
+ - **arte.tv:concert**
+ - **arte.tv:creative**
+ - **arte.tv:ddc**
+ - **arte.tv:embed**
+ - **arte.tv:future**
+ - **audiomack**
+ - **AUEngine**
+ - **Azubu**
+ - **bambuser**
+ - **bambuser:channel**
+ - **Bandcamp**
+ - **Bandcamp:album**
+ - **bbc.co.uk**: BBC iPlayer
+ - **Beeg**
+ - **BehindKink**
+ - **Bet**
+ - **Bild**: Bild.de
+ - **BiliBili**
+ - **blinkx**
+ - **blip.tv:user**
+ - **BlipTV**
+ - **Bloomberg**
+ - **Bpb**: Bundeszentrale für politische Bildung
+ - **BR**: Bayerischer Rundfunk Mediathek
+ - **Break**
+ - **Brightcove**
+ - **BuzzFeed**
+ - **BYUtv**
+ - **Canal13cl**
+ - **canalc2.tv**
+ - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
+ - **CBS**
+ - **CBSNews**: CBS News
+ - **CeskaTelevize**
+ - **channel9**: Channel 9
+ - **Chilloutzone**
+ - **Cinchcast**
+ - **Cinemassacre**
+ - **clipfish**
+ - **cliphunter**
+ - **Clipsyndicate**
+ - **Cloudy**
+ - **Clubic**
+ - **cmt.com**
+ - **CNET**
+ - **CNN**
+ - **CNNBlogs**
+ - **CollegeHumor**
+ - **ComCarCoff**
+ - **ComedyCentral**
+ - **ComedyCentralShows**: The Daily Show / The Colbert Report
+ - **CondeNast**: Condé Nast media group: Condé Nast, GQ, Glamour, Vanity Fair, Vogue, W Magazine, WIRED
+ - **Cracked**
+ - **Criterion**
+ - **Crunchyroll**
+ - **crunchyroll:playlist**
+ - **CSpan**: C-SPAN
+ - **culturebox.francetvinfo.fr**
+ - **dailymotion**
+ - **dailymotion:playlist**
+ - **dailymotion:user**
+ - **daum.net**
+ - **DBTV**
+ - **DeezerPlaylist**
+ - **defense.gouv.fr**
+ - **Discovery**
+ - **divxstage**: DivxStage
+ - **Dotsub**
+ - **Dropbox**
+ - **DrTuber**
+ - **DRTV**
+ - **Dump**
+ - **dvtv**: http://video.aktualne.cz/
+ - **EbaumsWorld**
+ - **eHow**
+ - **Einthusan**
+ - **eitb.tv**
+ - **EllenTV**
+ - **EllenTV:clips**
+ - **ElPais**: El País
+ - **EMPFlix**
+ - **Engadget**
+ - **Eporner**
+ - **Escapist**
+ - **EveryonesMixtape**
+ - **exfm**: ex.fm
+ - **ExpoTV**
+ - **ExtremeTube**
+ - **facebook**
+ - **faz.net**
+ - **fc2**
+ - **fernsehkritik.tv**
+ - **fernsehkritik.tv:postecke**
+ - **Firedrive**
+ - **Firstpost**
+ - **firsttv**: Видеоархив - Первый канал
+ - **Flickr**
+ - **Folketinget**: Folketinget (ft.dk; Danish parliament)
+ - **Foxgay**
+ - **FoxNews**
+ - **france2.fr:generation-quoi**
+ - **FranceCulture**
+ - **FranceInter**
+ - **francetv**: France 2, 3, 4, 5 and Ô
+ - **francetvinfo.fr**
+ - **Freesound**
+ - **freespeech.org**
+ - **FreeVideo**
+ - **FunnyOrDie**
+ - **Gamekings**
+ - **GameOne**
+ - **gameone:playlist**
+ - **GameSpot**
+ - **GameStar**
+ - **Gametrailers**
+ - **GDCVault**
+ - **generic**: Generic downloader that works on some sites
+ - **GiantBomb**
+ - **Glide**: Glide mobile video messages (glide.me)
+ - **Globo**
+ - **GodTube**
+ - **GoldenMoustache**
+ - **Golem**
+ - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in and fastvideo.in
+ - **Goshgay**
+ - **Grooveshark**
+ - **Groupon**
+ - **Hark**
+ - **Heise**
+ - **Helsinki**: helsinki.fi
+ - **HentaiStigma**
+ - **HornBunny**
+ - **HostingBulk**
+ - **HotNewHipHop**
+ - **Howcast**
+ - **HowStuffWorks**
+ - **HuffPost**: Huffington Post
+ - **Hypem**
+ - **Iconosquare**
+ - **ign.com**
+ - **imdb**: Internet Movie Database trailers
+ - **imdb:list**: Internet Movie Database lists
+ - **Ina**
+ - **InfoQ**
+ - **Instagram**
+ - **instagram:user**: Instagram user profile
+ - **InternetVideoArchive**
+ - **IPrima**
+ - **ivi**: ivi.ru
+ - **ivi:compilation**: ivi.ru compilations
+ - **Izlesene**
+ - **JadoreCettePub**
+ - **JeuxVideo**
+ - **Jove**
+ - **jpopsuki.tv**
+ - **Jukebox**
+ - **Kankan**
+ - **keek**
+ - **KeezMovies**
+ - **KhanAcademy**
+ - **KickStarter**
+ - **kontrtube**: KontrTube.ru - Труба зовёт
+ - **KrasView**: Красвью
+ - **Ku6**
+ - **la7.tv**
+ - **Laola1Tv**
+ - **lifenews**: LIFE | NEWS
+ - **LiveLeak**
+ - **livestream**
+ - **livestream:original**
+ - **lrt.lt**
+ - **lynda**: lynda.com videos
+ - **lynda:course**: lynda.com online courses
+ - **m6**
+ - **macgamestore**: MacGameStore trailers
+ - **mailru**: Видео@Mail.Ru
+ - **Malemotion**
+ - **MDR**
+ - **metacafe**
+ - **Metacritic**
+ - **Mgoon**
+ - **Minhateca**
+ - **MinistryGrid**
+ - **mitele.es**
+ - **mixcloud**
+ - **MLB**
+ - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
+ - **Mofosex**
+ - **Mojvideo**
+ - **Moniker**: allmyvideos.net and vidspot.net
+ - **mooshare**: Mooshare.biz
+ - **Morningstar**: morningstar.com
+ - **Motherless**
+ - **Motorsport**: motorsport.com
+ - **MovieClips**
+ - **Moviezine**
+ - **movshare**: MovShare
+ - **MPORA**
+ - **MTV**
+ - **mtviggy.com**
+ - **mtvservices:embedded**
+ - **MuenchenTV**: münchen.tv
+ - **MusicPlayOn**
+ - **MusicVault**
+ - **muzu.tv**
+ - **MySpace**
+ - **MySpace:album**
+ - **MySpass**
+ - **myvideo**
+ - **MyVidster**
+ - **Naver**
+ - **NBA**
+ - **NBC**
+ - **NBCNews**
+ - **ndr**: NDR.de - Mediathek
+ - **NDTV**
+ - **NerdCubedFeed**
+ - **Newgrounds**
+ - **Newstube**
+ - **nfb**: National Film Board of Canada
+ - **nfl.com**
+ - **nhl.com**
+ - **nhl.com:videocenter**: NHL videocenter category
+ - **niconico**: ニコニコ動画
+ - **NiconicoPlaylist**
+ - **Noco**
+ - **Normalboots**
+ - **NosVideo**
+ - **novamov**: NovaMov
+ - **Nowness**
+ - **nowvideo**: NowVideo
+ - **npo.nl**
+ - **NRK**
+ - **NRKTV**
+ - **NTV**
+ - **Nuvid**
+ - **NYTimes**
+ - **ocw.mit.edu**
+ - **OktoberfestTV**
+ - **on.aol.com**
+ - **Ooyala**
+ - **orf:oe1**: Radio Österreich 1
+ - **orf:tvthek**: ORF TVthek
+ - **ORFFM4**: radio FM4
+ - **parliamentlive.tv**: UK parliament videos
+ - **Patreon**
+ - **PBS**
+ - **Phoenix**
+ - **Photobucket**
+ - **PlanetaPlay**
+ - **play.fm**
+ - **played.to**
+ - **Playvid**
+ - **plus.google**: Google Plus
+ - **pluzz.francetv.fr**
+ - **podomatic**
+ - **PornHd**
+ - **PornHub**
+ - **Pornotube**
+ - **PornoXO**
+ - **PromptFile**
+ - **prosiebensat1**: ProSiebenSat.1 Digital
+ - **Pyvideo**
+ - **QuickVid**
+ - **radio.de**
+ - **radiofrance**
+ - **Rai**
+ - **RBMARadio**
+ - **RedTube**
+ - **Restudy**
+ - **ReverbNation**
+ - **RingTV**
+ - **RottenTomatoes**
+ - **Roxwel**
+ - **RTBF**
+ - **RTLnow**
+ - **rtlxl.nl**
+ - **RTP**
+ - **RTS**: RTS.ch
+ - **rtve.es:alacarta**: RTVE a la carta
+ - **rtve.es:live**: RTVE.es live streams
+ - **RUHD**
+ - **rutube**: Rutube videos
+ - **rutube:channel**: Rutube channels
+ - **rutube:movie**: Rutube movies
+ - **rutube:person**: Rutube person videos
+ - **RUTV**: RUTV.RU
+ - **Sapo**: SAPO Vídeos
+ - **savefrom.net**
+ - **SBS**: sbs.com.au
+ - **SciVee**
+ - **screen.yahoo:search**: Yahoo screen search
+ - **Screencast**
+ - **ScreencastOMatic**
+ - **ScreenwaveMedia**
+ - **ServingSys**
+ - **Sexu**
+ - **SexyKarma**: Sexy Karma and Watch Indian Porn
+ - **Shared**
+ - **ShareSix**
+ - **Sina**
+ - **Slideshare**
+ - **Slutload**
+ - **smotri**: Smotri.com
+ - **smotri:broadcast**: Smotri.com broadcasts
+ - **smotri:community**: Smotri.com community videos
+ - **smotri:user**: Smotri.com user videos
+ - **Snotr**
+ - **Sockshare**
+ - **Sohu**
+ - **soundcloud**
+ - **soundcloud:playlist**
+ - **soundcloud:set**
+ - **soundcloud:user**
+ - **Soundgasm**
+ - **southpark.cc.com**
+ - **southpark.de**
+ - **Space**
+ - **Spankwire**
+ - **Spiegel**
+ - **Spiegel:Article**: Articles on spiegel.de
+ - **Spiegeltv**
+ - **Spike**
+ - **Sport5**
+ - **SportBox**
+ - **SportDeutschland**
+ - **SRMediathek**: Süddeutscher Rundfunk
+ - **stanfordoc**: Stanford Open ClassRoom
+ - **Steam**
+ - **streamcloud.eu**
+ - **StreamCZ**
+ - **SunPorno**
+ - **SWRMediathek**
+ - **Syfy**
+ - **SztvHu**
+ - **Tagesschau**
+ - **Tapely**
+ - **Tass**
+ - **teachertube**: teachertube.com videos
+ - **teachertube:user:collection**: teachertube.com user and collection videos
+ - **TeachingChannel**
+ - **Teamcoco**
+ - **TeamFour**
+ - **TechTalks**
+ - **techtv.mit.edu**
+ - **TED**
+ - **tegenlicht.vpro.nl**
+ - **TeleBruxelles**
+ - **telecinco.es**
+ - **TeleMB**
+ - **TenPlay**
+ - **TF1**
+ - **TheOnion**
+ - **ThePlatform**
+ - **TheSixtyOne**
+ - **ThisAV**
+ - **THVideo**
+ - **THVideoPlaylist**
+ - **tinypic**: tinypic.com videos
+ - **tlc.com**
+ - **tlc.de**
+ - **TMZ**
+ - **TNAFlix**
+ - **tou.tv**
+ - **Toypics**: Toypics user profile
+ - **ToypicsUser**: Toypics user profile
+ - **TrailerAddict** (Currently broken)
+ - **Trilulilu**
+ - **TruTube**
+ - **Tube8**
+ - **Tudou**
+ - **Tumblr**
+ - **TuneIn**
+ - **Turbo**
+ - **Tutv**
+ - **tv.dfb.de**
+ - **tvigle**: Интернет-телевидение Tvigle.ru
+ - **tvp.pl**
+ - **TVPlay**: TV3Play and related services
+ - **Twitch**
+ - **Ubu**
+ - **udemy**
+ - **udemy:course**
+ - **Unistra**
+ - **Urort**: NRK P3 Urørt
+ - **ustream**
+ - **ustream:channel**
+ - **Vbox7**
+ - **VeeHD**
+ - **Veoh**
+ - **Vesti**: Вести.Ru
+ - **Vevo**
+ - **VGTV**
+ - **vh1.com**
+ - **Vice**
+ - **Viddler**
+ - **video.google:search**: Google Video search
+ - **video.mit.edu**
+ - **VideoBam**
+ - **VideoDetective**
+ - **videofy.me**
+ - **videolectures.net**
+ - **VideoMega**
+ - **VideoPremium**
+ - **VideoTt**: video.tt - Your True Tube
+ - **videoweed**: VideoWeed
+ - **Vidme**
+ - **Vidzi**
+ - **viki**
+ - **vimeo**
+ - **vimeo:album**
+ - **vimeo:channel**
+ - **vimeo:group**
+ - **vimeo:likes**: Vimeo user likes
+ - **vimeo:review**: Review pages on vimeo
+ - **vimeo:user**
+ - **vimeo:watchlater**: Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)
+ - **Vimple**: Vimple.ru
+ - **Vine**
+ - **vine:user**
+ - **vk.com**
+ - **vk.com:user-videos**: vk.com:All of a user's videos
+ - **Vodlocker**
+ - **Vporn**
+ - **VRT**
+ - **vube**: Vube.com
+ - **VuClip**
+ - **vulture.com**
+ - **Walla**
+ - **WashingtonPost**
+ - **wat.tv**
+ - **WayOfTheMaster**
+ - **WDR**
+ - **wdr:mobile**
+ - **WDRMaus**: Sendung mit der Maus
+ - **Weibo**
+ - **Wimp**
+ - **Wistia**
+ - **WorldStarHipHop**
+ - **wrzuta.pl**
+ - **XBef**
+ - **XboxClips**
+ - **XHamster**
+ - **XMinus**
+ - **XNXX**
+ - **XTube**
+ - **XTubeUser**: XTube user profile
+ - **XVideos**
+ - **Yahoo**: Yahoo screen and movies
+ - **YesJapan**
+ - **Ynet**
+ - **YouJizz**
+ - **Youku**
+ - **YouPorn**
+ - **YourUpload**
+ - **youtube**: YouTube.com
+ - **youtube:channel**: YouTube.com channels
+ - **youtube:favorites**: YouTube.com favourite videos, ":ytfav" for short (requires authentication)
+ - **youtube:history**: Youtube watch history, ":ythistory" for short (requires authentication)
+ - **youtube:playlist**: YouTube.com playlists
+ - **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication)
+ - **youtube:search**: YouTube.com searches
+ - **youtube:search:date**: YouTube.com searches, newest videos first
+ - **youtube:search_url**: YouTube.com search URLs
+ - **youtube:show**: YouTube.com (multi-season) shows
+ - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
+ - **youtube:toplist**: YouTube.com top lists, "yttoplist:{channel}:{list title}" (Example: "yttoplist:music:Top Tracks")
+ - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
+ - **youtube:watch_later**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
+ - **ZDF**
+ - **ZDFChannel**
+ - **zingmp3:album**: mp3.zing.vn albums
+ - **zingmp3:song**: mp3.zing.vn songs
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2b4d8c62f..40b2791c7 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -23,6 +23,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     unsmuggle_url,
+    UnsupportedError,
     url_basename,
 )
 from .brightcove import BrightcoveIE
@@ -1057,7 +1058,7 @@ class GenericIE(InfoExtractor):
                     'url': new_url,
                 }
         if not found:
-            raise ExtractorError('Unsupported URL: %s' % url)
+            raise UnsupportedError(url)
 
         entries = []
         for video_url in found:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 43b7c94ba..efbe64fb3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -464,6 +464,13 @@ class ExtractorError(Exception):
         return ''.join(traceback.format_tb(self.traceback))
 
 
+class UnsupportedError(ExtractorError):
+    def __init__(self, url):
+        super(UnsupportedError, self).__init__(
+            'Unsupported URL: %s' % url, expected=True)
+        self.url = url
+
+
 class RegexNotFoundError(ExtractorError):
     """Error when a regex didn't match"""
     pass

From 966ea3aebde5968c3ca1b655bac2e91f6bcea466 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 30 Dec 2014 23:41:29 +0100
Subject: [PATCH 330/974] [README] Typo / clarify FAQ

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 93a2750c2..915bcd0cd 100644
--- a/README.md
+++ b/README.md
@@ -453,9 +453,9 @@ To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistri
 
 For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
-It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which maches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
+It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 
-If you want to find out, simply call youtube-dl. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
+If you want to find out whether a given URL is supported, simply call youtube-dl with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
 
 # DEVELOPER INSTRUCTIONS
 

From 1931a73f392698d01d148f69e612391ad2d7138d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Dec 2014 18:03:51 +0600
Subject: [PATCH 331/974] [echomsk] Add extractor (Closes #4600)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/echomsk.py  | 46 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/echomsk.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c15786ad7..45b50792e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -99,6 +99,7 @@ from .discovery import DiscoveryIE
 from .divxstage import DivxStageIE
 from .dropbox import DropboxIE
 from .ebaumsworld import EbaumsWorldIE
+from .echomsk import EchoMskIE
 from .ehow import EHowIE
 from .eighttracks import EightTracksIE
 from .einthusan import EinthusanIE
diff --git a/youtube_dl/extractor/echomsk.py b/youtube_dl/extractor/echomsk.py
new file mode 100644
index 000000000..d2d94049d
--- /dev/null
+++ b/youtube_dl/extractor/echomsk.py
@@ -0,0 +1,46 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class EchoMskIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?echo\.msk\.ru/sounds/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.echo.msk.ru/sounds/1464134.html',
+        'md5': '2e44b3b78daff5b458e4dbc37f191f7c',
+        'info_dict': {
+            'id': '1464134',
+            'ext': 'mp3',
+            'title': 'Особое мнение - 29 декабря 2014, 19:08',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        audio_url = self._search_regex(
+            r'<a rel="mp3" href="([^"]+)">', webpage, 'audio URL')
+
+        title = self._html_search_regex(
+            r'<a href="/programs/[^"]+" target="_blank">([^<]+)</a>',
+            webpage, 'title')
+
+        air_date = self._html_search_regex(
+            r'(?s)<div class="date">(.+?)</div>',
+            webpage, 'date', fatal=False, default=None)
+
+        if air_date:
+            air_date = re.sub(r'(\s)\1+', r'\1', air_date)
+            if air_date:
+                title = '%s - %s' % (title, air_date)
+
+        return {
+            'id': video_id,
+            'url': audio_url,
+            'title': title,
+        }

From 7d900ef1bf131fc140d35d306500a5cec9565f89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 31 Dec 2014 14:15:16 +0100
Subject: [PATCH 332/974] [youtube] Add support for automatically translated
 subtitles (fixes #4555)

They have a manually uploaded subtitles track and YouTube can transtale it.
---
 test/test_subtitles.py          | 8 ++++++++
 youtube_dl/extractor/youtube.py | 5 +++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 7c4cd8218..d34565191 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -88,6 +88,14 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertTrue(subtitles['it'] is not None)
 
+    def test_youtube_translated_subtitles(self):
+        # This video has a subtitles track, which can be translated
+        self.url = 'Ky9eprVWzlI'
+        self.DL.params['writeautomaticsub'] = True
+        self.DL.params['subtitleslangs'] = ['it']
+        subtitles = self.getSubtitles()
+        self.assertTrue(subtitles['it'] is not None)
+
     def test_youtube_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'n5BB19UTcdA'
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3da83e3a8..224f1b041 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -662,10 +662,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             list_url = caption_url + '&' + list_params
             caption_list = self._download_xml(list_url, video_id)
             original_lang_node = caption_list.find('track')
-            if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr':
+            if original_lang_node is None:
                 self._downloader.report_warning('Video doesn\'t have automatic captions')
                 return {}
             original_lang = original_lang_node.attrib['lang_code']
+            caption_kind = original_lang_node.attrib.get('kind', '')
 
             sub_lang_list = {}
             for lang_node in caption_list.findall('target'):
@@ -675,7 +676,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     'tlang': sub_lang,
                     'fmt': sub_format,
                     'ts': timestamp,
-                    'kind': 'asr',
+                    'kind': caption_kind,
                 })
                 sub_lang_list[sub_lang] = caption_url + '&' + params
             return sub_lang_list

From b8bc7a696ba15cbc6d45b20320d928340246b508 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Dec 2014 19:40:35 +0600
Subject: [PATCH 333/974] [openfilm] Add extractor (Closes #4538)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/openfilm.py | 70 ++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 youtube_dl/extractor/openfilm.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 45b50792e..6c5827f88 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -299,6 +299,7 @@ from .nytimes import NYTimesIE
 from .nuvid import NuvidIE
 from .oktoberfesttv import OktoberfestTVIE
 from .ooyala import OoyalaIE
+from .openfilm import OpenFilmIE
 from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
diff --git a/youtube_dl/extractor/openfilm.py b/youtube_dl/extractor/openfilm.py
new file mode 100644
index 000000000..2249657eb
--- /dev/null
+++ b/youtube_dl/extractor/openfilm.py
@@ -0,0 +1,70 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    compat_urllib_parse,
+    parse_age_limit,
+    int_or_none,
+)
+
+
+class OpenFilmIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)openfilm\.com/videos/(?P<id>.+)'
+    _TEST = {
+        'url': 'http://www.openfilm.com/videos/human-resources-remastered',
+        'md5': '42bcd88c2f3ec13b65edf0f8ad1cac37',
+        'info_dict': {
+            'id': '32736',
+            'display_id': 'human-resources-remastered',
+            'ext': 'mp4',
+            'title': 'Human Resources (Remastered)',
+            'description': 'Social Engineering in the 20th Century.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 7164,
+            'timestamp': 1334756988,
+            'upload_date': '20120418',
+            'uploader_id': '41117',
+            'view_count': int,
+            'age_limit': 0,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        player = compat_urllib_parse.unquote_plus(
+            self._og_search_video_url(webpage))
+
+        video = json.loads(self._search_regex(
+            r'\bp=({.+?})(?:&|$)', player, 'video JSON'))
+
+        video_url = '%s1.mp4' % video['location']
+        video_id = video.get('video_id')
+        display_id = video.get('alias') or display_id
+        title = video.get('title')
+        description = video.get('description')
+        thumbnail = video.get('main_thumb')
+        duration = int_or_none(video.get('duration'))
+        timestamp = parse_iso8601(video.get('dt_published'), ' ')
+        uploader_id = video.get('user_id')
+        view_count = int_or_none(video.get('views_count'))
+        age_limit = parse_age_limit(video.get('age_limit'))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'uploader_id': uploader_id,
+            'view_count': view_count,
+            'age_limit': age_limit,
+        }

From 60e47a2699bea32f12a92e59d02d1ba0e9cbdec9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 31 Dec 2014 15:44:15 +0100
Subject: [PATCH 334/974] [youtube] Use '_download_xml' for getting the
 available subtitles

---
 youtube_dl/extractor/youtube.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 224f1b041..7f5aeb25b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -612,24 +612,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _get_available_subtitles(self, video_id, webpage):
         try:
-            sub_list = self._download_webpage(
+            subs_doc = self._download_xml(
                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
             self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
             return {}
-        lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
 
         sub_lang_list = {}
-        for l in lang_list:
-            lang = l[1]
+        for track in subs_doc.findall('track'):
+            lang = track.attrib['lang_code']
             if lang in sub_lang_list:
                 continue
             params = compat_urllib_parse.urlencode({
                 'lang': lang,
                 'v': video_id,
                 'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
-                'name': unescapeHTML(l[0]).encode('utf-8'),
+                'name': track.attrib['name'].encode('utf-8'),
             })
             url = 'https://www.youtube.com/api/timedtext?' + params
             sub_lang_list[lang] = url

From f86d543ebb6eafa1ca939a17601307bf29b88f6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 31 Dec 2014 17:24:14 +0100
Subject: [PATCH 335/974] [pbs] Catch geoblocking errors (closes #4516)

---
 youtube_dl/extractor/pbs.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6118ed5c2..afce732e1 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     unified_strdate,
     US_RATINGS,
 )
@@ -151,6 +152,19 @@ class PBSIE(InfoExtractor):
         info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id
         info = self._download_json(info_url, display_id)
 
+        redirect_url = info['alternate_encoding']['url']
+        redirect_info = self._download_json(
+            redirect_url + '?format=json', display_id,
+            'Downloading video url info')
+        if redirect_info['status'] == 'error':
+            if redirect_info['http_code'] == 403:
+                message = (
+                    'The video is not available in your region due to '
+                    'right restrictions')
+            else:
+                message = redirect_info['message']
+            raise ExtractorError(message, expected=True)
+
         rating_str = info.get('rating')
         if rating_str is not None:
             rating_str = rating_str.rpartition('-')[2]
@@ -160,7 +174,7 @@ class PBSIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'title': info['title'],
-            'url': info['alternate_encoding']['url'],
+            'url': redirect_info['url'],
             'ext': 'mp4',
             'description': info['program'].get('description'),
             'thumbnail': info.get('image_url'),

From 7b24bbdf49f4ad17a1cb134f9bf6a1075cb3f655 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Dec 2014 23:59:16 +0600
Subject: [PATCH 336/974] [xboxclips] Fix extraction

---
 youtube_dl/extractor/xboxclips.py | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/xboxclips.py b/youtube_dl/extractor/xboxclips.py
index a9aa72e73..9cf867807 100644
--- a/youtube_dl/extractor/xboxclips.py
+++ b/youtube_dl/extractor/xboxclips.py
@@ -1,46 +1,42 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
-    parse_iso8601,
-    float_or_none,
     int_or_none,
+    parse_filesize,
+    unified_strdate,
 )
 
 
 class XboxClipsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?xboxclips\.com/video\.php\?.*vid=(?P<id>[\w-]{36})'
+    _VALID_URL = r'https?://(?:www\.)?xboxclips\.com/(?:video\.php\?.*vid=|[^/]+/)(?P<id>[\w-]{36})'
     _TEST = {
         'url': 'https://xboxclips.com/video.php?uid=2533274823424419&gamertag=Iabdulelah&vid=074a69a9-5faf-46aa-b93b-9909c1720325',
         'md5': 'fbe1ec805e920aeb8eced3c3e657df5d',
         'info_dict': {
             'id': '074a69a9-5faf-46aa-b93b-9909c1720325',
             'ext': 'mp4',
-            'title': 'Iabdulelah playing Upload Studio',
-            'filesize_approx': 28101836.8,
-            'timestamp': 1407388500,
+            'title': 'Iabdulelah playing Titanfall',
+            'filesize_approx': 26800000,
             'upload_date': '20140807',
             'duration': 56,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._html_search_regex(
-            r'>Link: <a href="([^"]+)">', webpage, 'video URL')
+            r'>(?:Link|Download): <a href="([^"]+)">', webpage, 'video URL')
         title = self._html_search_regex(
             r'<title>XboxClips \| ([^<]+)</title>', webpage, 'title')
-        timestamp = parse_iso8601(self._html_search_regex(
+        upload_date = unified_strdate(self._html_search_regex(
             r'>Recorded: ([^<]+)<', webpage, 'upload date', fatal=False))
-        filesize = float_or_none(self._html_search_regex(
-            r'>Size: ([\d\.]+)MB<', webpage, 'file size', fatal=False), invscale=1024 * 1024)
+        filesize = parse_filesize(self._html_search_regex(
+            r'>Size: ([^<]+)<', webpage, 'file size', fatal=False))
         duration = int_or_none(self._html_search_regex(
             r'>Duration: (\d+) Seconds<', webpage, 'duration', fatal=False))
         view_count = int_or_none(self._html_search_regex(
@@ -50,7 +46,7 @@ class XboxClipsIE(InfoExtractor):
             'id': video_id,
             'url': video_url,
             'title': title,
-            'timestamp': timestamp,
+            'upload_date': upload_date,
             'filesize_approx': filesize,
             'duration': duration,
             'view_count': view_count,

From 079d1dcd8033b7c1470ad84899967d787cf1dec7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:11:56 +0600
Subject: [PATCH 337/974] [tnaflix] Fix title extraction

---
 youtube_dl/extractor/tnaflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 0ecd695f8..b9179ca80 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -12,7 +12,7 @@ from ..utils import (
 class TNAFlixIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[\w-]+)/(?P<display_id>[\w-]+)/video(?P<id>\d+)'
 
-    _TITLE_REGEX = None
+    _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
     _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
     _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
 

From 68f705cac53510dc272eebfe83694a2cbf2c2b46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:12:41 +0600
Subject: [PATCH 338/974] [tnaflix] Make sure config URL has correct scheme

---
 youtube_dl/extractor/tnaflix.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index b9179ca80..d48cbbf14 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -49,8 +49,8 @@ class TNAFlixIE(InfoExtractor):
         if duration:
             duration = parse_duration(duration[1:])
 
-        cfg_url = self._html_search_regex(
-            self._CONFIG_REGEX, webpage, 'flashvars.config')
+        cfg_url = self._proto_relative_url(self._html_search_regex(
+            self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')
 
         cfg_xml = self._download_xml(
             cfg_url, display_id, note='Downloading metadata',

From b7a7319c388e3a33851d87b589586abb6796a0e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:26:19 +0600
Subject: [PATCH 339/974] [slideshare] Fix extraction

---
 youtube_dl/extractor/slideshare.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index e7d776e7b..9f79ff5c1 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -30,7 +30,7 @@ class SlideshareIE(InfoExtractor):
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
         slideshare_obj = self._search_regex(
-            r'var slideshare_object =  ({.*?}); var user_info =',
+            r'var\s+slideshare_object\s*=\s*({.*?});\s*var\s+user_info\s*=',
             webpage, 'slideshare object')
         info = json.loads(slideshare_obj)
         if info['slideshow']['type'] != 'video':
@@ -41,7 +41,7 @@ class SlideshareIE(InfoExtractor):
         ext = info['jsplayer']['video_extension']
         video_url = compat_urlparse.urljoin(bucket, doc + '-SD.' + ext)
         description = self._html_search_regex(
-            r'<p\s+(?:style="[^"]*"\s+)?class=".*?description.*?"[^>]*>(.*?)</p>', webpage,
+            r'(?s)<p[^>]+itemprop="description"[^>]*>(.+?)</p>', webpage,
             'description', fatal=False)
 
         return {

From 6c6f1408f264f30fc4e2970e958cb7f9dc470d57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:37:14 +0600
Subject: [PATCH 340/974] [extractor/common] Allow multiline content tags

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4b950e485..6e264f687 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -589,7 +589,7 @@ class InfoExtractor(object):
         if display_name is None:
             display_name = name
         return self._html_search_regex(
-            r'''(?ix)<meta
+            r'''(?isx)<meta
                     (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
                     [^>]+content=(["\'])(?P<content>.*?)\1''' % re.escape(name),
             html, display_name, fatal=fatal, group='content', **kwargs)

From 5878e6398ce7945ccbb141a38c815667f8a1fe9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:37:57 +0600
Subject: [PATCH 341/974] [nrktv] Update tests' checksums

---
 youtube_dl/extractor/nrk.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 96f0ae1eb..43e8e619f 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -77,7 +77,7 @@ class NRKTVIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
-            'md5': '7b96112fbae1faf09a6f9ae1aff6cb84',
+            'md5': 'adf2c5454fa2bf032f47a9f8fb351342',
             'info_dict': {
                 'id': 'MUHH48000314',
                 'ext': 'flv',
@@ -89,7 +89,7 @@ class NRKTVIE(InfoExtractor):
         },
         {
             'url': 'http://tv.nrk.no/program/mdfp15000514',
-            'md5': 'af01795a31f1cf7265c8657534d8077b',
+            'md5': '383650ece2b25ecec996ad7b5bb2a384',
             'info_dict': {
                 'id': 'mdfp15000514',
                 'ext': 'flv',

From a1e9e6440f60f82b8d8552053219e1e8e3149c5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:46:03 +0600
Subject: [PATCH 342/974] [moevideo] Skip removed video test

---
 youtube_dl/extractor/moevideo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dl/extractor/moevideo.py
index 184f9c2c9..5a66302f6 100644
--- a/youtube_dl/extractor/moevideo.py
+++ b/youtube_dl/extractor/moevideo.py
@@ -52,7 +52,8 @@ class MoeVideoIE(InfoExtractor):
                 'height': 296,
                 'duration': 6027,
                 'filesize': 588257923,
-            }
+            },
+            'skip': 'Video has been removed',
         },
     ]
 

From 02ec32a1efc18a4f09b2422dc8c5bfb397a7e6ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 20:01:55 +0600
Subject: [PATCH 343/974] [ceskatelevize] Adapt to new API (Closes #4531)

---
 youtube_dl/extractor/ceskatelevize.py | 88 +++++++++++----------------
 1 file changed, 37 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 2f866f3ef..ba8376338 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    float_or_none,
 )
 
 
@@ -19,41 +20,33 @@ class CeskaTelevizeIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.ceskatelevize.cz/ivysilani/10532695142-prvni-republika/213512120230004-spanelska-chripka',
+            'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
             'info_dict': {
-                'id': '213512120230004',
-                'ext': 'flv',
-                'title': 'První republika: Španělská chřipka',
-                'duration': 3107.4,
+                'id': '214411058091220',
+                'ext': 'mp4',
+                'title': 'Hyde Park Civilizace',
+                'description': 'Věda a současná civilizace. Interaktivní pořad - prostor pro vaše otázky a komentáře',
+                'thumbnail': 're:^https?://.*\.jpg',
+                'duration': 3350,
             },
             'params': {
-                'skip_download': True,  # requires rtmpdump
+                # m3u8 download
+                'skip_download': True,
             },
-            'skip': 'Works only from Czech Republic.',
-        },
-        {
-            'url': 'http://www.ceskatelevize.cz/ivysilani/1030584952-tsatsiki-maminka-a-policajt',
-            'info_dict': {
-                'id': '20138143440',
-                'ext': 'flv',
-                'title': 'Tsatsiki, maminka a policajt',
-                'duration': 6754.1,
-            },
-            'params': {
-                'skip_download': True,  # requires rtmpdump
-            },
-            'skip': 'Works only from Czech Republic.',
         },
         {
             'url': 'http://www.ceskatelevize.cz/ivysilani/10532695142-prvni-republika/bonus/14716-zpevacka-z-duparny-bobina',
             'info_dict': {
                 'id': '14716',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'První republika: Zpěvačka z Dupárny Bobina',
-                'duration': 90,
+                'description': 'Sága mapující atmosféru první republiky od r. 1918 do r. 1945.',
+                'thumbnail': 're:^https?://.*\.jpg',
+                'duration': 88.4,
             },
             'params': {
-                'skip_download': True,  # requires rtmpdump
+                # m3u8 download
+                'skip_download': True,
             },
         },
     ]
@@ -80,8 +73,9 @@ class CeskaTelevizeIE(InfoExtractor):
             'requestSource': 'iVysilani',
         }
 
-        req = compat_urllib_request.Request('http://www.ceskatelevize.cz/ivysilani/ajax/get-playlist-url',
-                                            data=compat_urllib_parse.urlencode(data))
+        req = compat_urllib_request.Request(
+            'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
+            data=compat_urllib_parse.urlencode(data))
 
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         req.add_header('x-addr', '127.0.0.1')
@@ -90,39 +84,31 @@ class CeskaTelevizeIE(InfoExtractor):
 
         playlistpage = self._download_json(req, video_id)
 
-        req = compat_urllib_request.Request(compat_urllib_parse.unquote(playlistpage['url']))
+        playlist_url = playlistpage['url']
+        if playlist_url == 'error_region':
+            raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
+
+        req = compat_urllib_request.Request(compat_urllib_parse.unquote(playlist_url))
         req.add_header('Referer', url)
 
-        playlist = self._download_xml(req, video_id)
+        playlist = self._download_json(req, video_id)
 
+        item = playlist['playlist'][0]
         formats = []
-        for i in playlist.find('smilRoot/body'):
-            if 'AD' not in i.attrib['id']:
-                base_url = i.attrib['base']
-                parsedurl = compat_urllib_parse_urlparse(base_url)
-                duration = i.attrib['duration']
-
-                for video in i.findall('video'):
-                    if video.attrib['label'] != 'AD':
-                        format_id = video.attrib['label']
-                        play_path = video.attrib['src']
-                        vbr = int(video.attrib['system-bitrate'])
-
-                        formats.append({
-                            'format_id': format_id,
-                            'url': base_url,
-                            'vbr': vbr,
-                            'play_path': play_path,
-                            'app': parsedurl.path[1:] + '?' + parsedurl.query,
-                            'rtmp_live': True,
-                            'ext': 'flv',
-                        })
-
+        for format_id, stream_url in item['streamUrls'].items():
+            formats.extend(self._extract_m3u8_formats(stream_url, video_id, 'mp4'))
         self._sort_formats(formats)
 
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        duration = float_or_none(item.get('duration'))
+        thumbnail = item.get('previewImageUrl')
+
         return {
             'id': episode_id,
-            'title': self._html_search_regex(r'<title>(.+?) — iVysílání — Česká televize</title>', webpage, 'title'),
-            'duration': float(duration),
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
             'formats': formats,
         }

From 9a237b776caa66fc6d1f6986c8f58599a1711aec Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 21:41:42 +0100
Subject: [PATCH 344/974] release 2015.01.01

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1420af746..89707ceef 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.17.2'
+__version__ = '2015.01.01'

From a42419da42808f8b4a82efa4918c8d09afb8ff39 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 22:01:26 +0100
Subject: [PATCH 345/974] [options] Upper-case options and URL in --help output

Hopefully, this reduces confusion as in #4610.
---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 21c452141..a018c1d71 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -109,7 +109,7 @@ def parseOpts(overrideArguments=None):
     kw = {
         'version': __version__,
         'formatter': fmt,
-        'usage': '%prog [options] url [url...]',
+        'usage': '%prog [OPTIONS] URL [URL...]',
         'conflict_handler': 'resolve',
     }
 

From 931e2d1d26f32c0ae3c1508b664207654529344d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 22:15:46 +0100
Subject: [PATCH 346/974] [bbccouk] PEP8

---
 youtube_dl/extractor/bbccouk.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index f690dc803..53aea0ec9 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -241,8 +241,8 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
 
         # fallback to legacy playlist
         playlist = self._download_xml(
-                'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id,
-                playlist_id, 'Downloading legacy playlist XML')
+            'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id,
+            playlist_id, 'Downloading legacy playlist XML')
 
         no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
         if no_items is not None:

From 9f435c5f1ce9566cf69b582b98b941d980d61262 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 22:34:58 +0100
Subject: [PATCH 347/974] Add an extractor for common mistakes (#4610)

---
 youtube_dl/extractor/__init__.py       |  1 +
 youtube_dl/extractor/commonmistakes.py | 29 ++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 youtube_dl/extractor/commonmistakes.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6c5827f88..a95de54cb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -71,6 +71,7 @@ from .cnn import (
 from .collegehumor import CollegeHumorIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
+from .commonmistakes import CommonMistakesIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py
new file mode 100644
index 000000000..75c06903f
--- /dev/null
+++ b/youtube_dl/extractor/commonmistakes.py
@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class CommonMistakesIE(InfoExtractor):
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'''(?x)
+        (?:url|URL)
+    '''
+
+    _TESTS = [{
+        'url': 'url',
+        'only_matching': True,
+    }, {
+        'url': 'URL',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        msg = (
+            'You\'ve asked youtube-dl to download the URL "%s". '
+            'That doesn\'t make any sense. '
+            'Simply remove the parameter in your command or configuration.'
+        ) % url
+        if self._downloader.params.get('verbose'):
+            msg += ' Add -v to the command line to see what arguments and configuration youtube-dl got.'
+        raise ExtractorError(msg, expected=True)

From 50a0f6df7e058c466d668d033a358b7e02a2f80f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 22:47:21 +0100
Subject: [PATCH 348/974] [/__init__] Add another cute search example

---
 youtube_dl/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index e79320323..56f560d26 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -112,7 +112,7 @@ def _real_main(argv=None):
             if desc is False:
                 continue
             if hasattr(ie, 'SEARCH_KEY'):
-                _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny')
+                _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow')
                 _COUNTS = ('', '5', '10', 'all')
                 desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
             compat_print(desc)

From 772fd5cc44c3682aed4a166936f7bf8ccccbbea4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 23:44:39 +0100
Subject: [PATCH 349/974] [youtube] Add a pseudo-extractor for truncated
 YouTube video IDs (#4610)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/youtube.py  | 17 +++++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a95de54cb..9848ff611 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -545,6 +545,7 @@ from .youtube import (
     YoutubeShowIE,
     YoutubeSubscriptionsIE,
     YoutubeTopListIE,
+    YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeUserIE,
     YoutubeWatchLaterIE,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7f5aeb25b..8c7842ee8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1701,3 +1701,20 @@ class YoutubeTruncatedURLIE(InfoExtractor):
             '"http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
             ' or simply  youtube-dl BaW_jenozKc  .',
             expected=True)
+
+
+class YoutubeTruncatedIDIE(InfoExtractor):
+    IE_NAME = 'youtube:truncated_id'
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'https?://(?:www\.)youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
+
+    _TESTS = [{
+        'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        raise ExtractorError(
+            'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
+            expected=True)

From defaf19f5de839777cda1654d42b54850a78d8a7 Mon Sep 17 00:00:00 2001
From: Your Name <you@example.com>
Date: Fri, 2 Jan 2015 02:20:04 -0600
Subject: [PATCH 350/974] Push api updates to simplify audiomack, add support
 for albums

---
 youtube_dl/extractor/__init__.py  |   2 +-
 youtube_dl/extractor/audiomack.py | 110 ++++++++++++++++++++++--------
 2 files changed, 84 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6c5827f88..e4c51f238 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -26,7 +26,7 @@ from .arte import (
     ArteTVEmbedIE,
 )
 from .atresplayer import AtresPlayerIE
-from .audiomack import AudiomackIE
+from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .auengine import AUEngineIE
 from .azubu import AzubuIE
 from .bambuser import BambuserIE, BambuserChannelIE
diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 622b20989..b6f3039c7 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -9,20 +9,21 @@ import time
 
 
 class AudiomackIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P<id>[\w/-]+)'
+    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/(song)/(?P<id>[\w/-]+)'
     IE_NAME = 'audiomack'
     _TESTS = [
-        # hosted on audiomack
+        # audiomack
         {
             'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary',
             'info_dict':
             {
-                'id': 'roosh-williams/extraordinary',
-                'ext': 'mp3',
-                'title': 'Roosh Williams - Extraordinary'
+                'id': '310086',
+                "ext": "mp3",
+                "artist": "Roosh Williams",
+                'title': 'Extraordinary'
             }
         },
-        # hosted on soundcloud via audiomack
+        # audiomack through soundcloud
         {
             'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
@@ -37,33 +38,88 @@ class AudiomackIE(InfoExtractor):
         },
     ]
 
+    @staticmethod
+    def create_song_dictionary(api_response, album_url_tag, track_no=0):
+        # All keys are the same in audiomack api and InfoExtractor format
+        entry = {key: api_response[key] for key in ["title", "artist", "id", "url"] if key in api_response}
+        # Fudge values in the face of missing metadata
+        if "id" not in entry:
+            entry["id"] = track_no
+        if "title" not in entry:
+            entry["title"] = album_url_tag
+        return entry
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        # URLs end with [uploader name]/[uploader title]
+        # this title is whatever the user types in, and is rarely
+        # the proper song title.  Real metadata is in the api response
+        album_url_tag = self._match_id(url)
 
+        # Request the extended version of the api for extra fields like artist and title
         api_response = self._download_json(
-            "http://www.audiomack.com/api/music/url/song/%s?_=%d" % (
-                video_id, time.time()),
-            video_id)
+            "http://www.audiomack.com/api/music/url/song/%s?extended=1&_=%d" % (
+                album_url_tag, time.time()),
+            album_url_tag)
 
-        if "url" not in api_response:
-            raise ExtractorError("Unable to deduce api url of song")
-        realurl = api_response["url"]
+        # API is inconsistent with errors
+        if "url" not in api_response or not api_response["url"] or "error" in api_response:
+            raise ExtractorError("Invalid url %s", url)
 
         # Audiomack wraps a lot of soundcloud tracks in their branded wrapper
-        # - if so, pass the work off to the soundcloud extractor
-        if SoundcloudIE.suitable(realurl):
-            return {'_type': 'url', 'url': realurl, 'ie_key': 'Soundcloud'}
+        # if so, pass the work off to the soundcloud extractor
+        if SoundcloudIE.suitable(api_response["url"]):
+            return {'_type': 'url', 'url': api_response["url"], 'ie_key': 'Soundcloud'}
 
-        webpage = self._download_webpage(url, video_id)
-        artist = self._html_search_regex(
-            r'<span class="artist">(.*?)</span>', webpage, "artist")
-        songtitle = self._html_search_regex(
-            r'<h1 class="profile-title song-title"><span class="artist">.*?</span>(.*?)</h1>',
-            webpage, "title")
-        title = artist + " - " + songtitle
+        return self.create_song_dictionary(api_response, album_url_tag)
 
-        return {
-            'id': video_id,
-            'title': title,
-            'url': realurl,
+
+class AudiomackAlbumIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/album/(?P<id>[\w/-]+)'
+    IE_NAME = 'audiomack:album'
+    _TESTS = [
+        # Standard album playlist
+        {
+            'url': 'http://www.audiomack.com/album/flytunezcom/tha-tour-part-2-mixtape',
+            "playlist_count": 15,
+            'info_dict':
+            {
+                'id': "812251",
+                'title': "Tha Tour: Part 2 (Official Mixtape)"
+            }
+        },
+        # Album playlist ripped from fakeshoredrive with no metadata
+        {
+            "url": "http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project",
+            "playlist_count": 10
         }
+    ]
+
+    def _real_extract(self, url):
+        # URLs end with [uploader name]/[uploader title]
+        # this title is whatever the user types in, and is rarely
+        # the proper song title.  Real metadata is in the api response
+        album_url_tag = self._match_id(url)
+        result = {"_type": "playlist", "entries": []}
+        # There is no one endpoint for album metadata - instead it is included/repeated in each song's metadata
+        # Therefore we don't know how many songs the album has and must infi-loop until failure
+        track_no = 0
+        while True:
+            # Get song's metadata
+            api_response = self._download_json("http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d"
+                                               % (album_url_tag, track_no, time.time()), album_url_tag)
+
+            # Total failure, only occurs when url is totally wrong
+            # Won't happen in middle of valid playlist (next case)
+            if "url" not in api_response or "error" in api_response:
+                raise ExtractorError("Invalid url for track %d of album url %s" % (track_no, url))
+            # URL is good but song id doesn't exist - usually means end of playlist
+            elif not api_response["url"]:
+                break
+            else:
+                # Pull out the album metadata and add to result (if it exists)
+                for resultkey, apikey in [("id", "album_id"), ("title", "album_title")]:
+                    if apikey in api_response and resultkey not in result:
+                        result[resultkey] = api_response[apikey]
+                result["entries"].append(AudiomackIE.create_song_dictionary(api_response, album_url_tag, track_no))
+            track_no += 1
+        return result

From 5b9aefef773463977504081faa1db88bd0e94300 Mon Sep 17 00:00:00 2001
From: dwemthy <dwemthy@users.noreply.github.com>
Date: Fri, 2 Jan 2015 11:52:48 +0000
Subject: [PATCH 351/974] [xhamster] Add HTTPS support

---
 youtube_dl/extractor/xhamster.py | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 6b37bcbc9..519bfc191 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class XHamsterIE(InfoExtractor):
     """Information Extractor for xHamster"""
-    _VALID_URL = r'http://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
+    _VALID_URL = r'(?P<proto>https?)://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
     _TESTS = [
         {
             'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',
@@ -39,6 +39,30 @@ class XHamsterIE(InfoExtractor):
                 'duration': 200,
                 'age_limit': 18,
             }
+        },
+        {
+            'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',
+            'info_dict': {
+                'id': '2272726',
+                'ext': 'mp4',
+                'title': 'Amber slayed by the Knight',
+                'upload_date': '20131009',
+                'uploader_id': 'amberblank',
+                'duration': 149,
+                'age_limit': 18,
+            }
+        },
+        {
+            "url": "https://xhamster.com/movies/1444747/hotkinkyjo_amp_asian_girl_elbow_deep_anal_fisting_and_footing.html",
+            'info_dict': {
+                "id": "1444747",
+                'ext': 'mp4',
+                "title": "HOTKINKYJO & ASIAN GIRL ELBOW DEEP ANAL FISTING AND FOOTING",
+                "upload_date": "20120922",
+                "uploader_id": "alex1981",
+                "duration": 80,
+                "age_limit": 18,
+            }
         }
     ]
 
@@ -57,7 +81,8 @@ class XHamsterIE(InfoExtractor):
 
         video_id = mobj.group('id')
         seo = mobj.group('seo')
-        mrss_url = 'http://xhamster.com/movies/%s/%s.html' % (video_id, seo)
+        proto = mobj.group('proto')
+        mrss_url = '%s://xhamster.com/movies/%s/%s.html' % (proto, video_id, seo)
         webpage = self._download_webpage(mrss_url, video_id)
 
         title = self._html_search_regex(r'<title>(?P<title>.+?) - xHamster\.com</title>', webpage, 'title')

From f58487b392f7943c31e41b200bef8e97822529db Mon Sep 17 00:00:00 2001
From: Thijs Vermeir <thijsvermeir@gmail.com>
Date: Fri, 2 Jan 2015 13:13:18 +0100
Subject: [PATCH 352/974] [vier] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vier.py     | 75 ++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 youtube_dl/extractor/vier.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9848ff611..9ccd1b32e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -474,6 +474,7 @@ from .videott import VideoTtIE
 from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
 from .vidzi import VidziIE
+from .vier import VierIE, VierVideosIE
 from .vimeo import (
     VimeoIE,
     VimeoAlbumIE,
diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
new file mode 100644
index 000000000..e3f1b4714
--- /dev/null
+++ b/youtube_dl/extractor/vier.py
@@ -0,0 +1,75 @@
+from .common import InfoExtractor
+from ..utils import escape_url
+
+import re
+
+class VierIE (InfoExtractor):
+  _VALID_URL = r'(?:http://)?www.vier.be/(?P<program>.*)/videos/(.+?)/(?P<id>\d*)'
+  _TEST = {
+      'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
+      'md5': 'bf48f4eb998cbde44ecd02fc42c51149',
+      'info_dict': {
+          'id': '16129',
+          'ext': 'mp4',
+          'title': 'Het wordt warm in De Moestuin',
+          'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
+      },
+  }
+
+  def _real_extract (self, url):
+    mobj = re.match (self._VALID_URL, url)
+
+    program = mobj.group ('program')
+    video_id = mobj.group ('id')
+
+    webpage = self._download_webpage (url, video_id)
+
+    title = self._html_search_regex(r'<meta property="og:title" content="(.+?)" />', webpage, u'title')
+    description = self._html_search_regex (r'<meta property="og:description" content="(.+?)" />', webpage, u'description')
+    vod_id =  self._html_search_regex(r'"filename" : "(.+?)"', webpage, u'playlist URL')
+    url = escape_url ("http://vod.streamcloud.be/vier_vod/mp4:_definst_/" + vod_id + ".mp4/playlist.m3u8")
+
+    return {
+        'id': video_id,
+        'title': title,
+        'description': description,
+        'formats': self._extract_m3u8_formats(url, video_id, 'mp4'),
+    }
+
+class VierVideosIE (InfoExtractor):
+  _VALID_URL = r'http://www.vier.be/(?P<program>.*)/videos(\?page=(?P<page>\d*))?$'
+  _TESTS = [{
+    'url': 'http://www.vier.be/demoestuin/videos',
+    'info_dict': {
+      'id': 'demoestuin page(0)',
+    },
+    'playlist_mincount': 20,
+  },
+  {
+    'url': 'http://www.vier.be/demoestuin/videos?page=6',
+    'info_dict': {
+      'id': 'demoestuin page(6)',
+    },
+    'playlist_mincount': 20,
+  }]
+
+  def _real_extract (self, url):
+    mobj = re.match (self._VALID_URL, url)
+
+    program = mobj.group ('program')
+    page = mobj.group ('page')
+    if page == None:
+      page = 0
+
+    videos_id = program + " page(" + str (page) + ")"
+    videos_page = self._download_webpage (url, videos_id, note='Retrieving videos page')
+
+    return {
+        '_type': 'playlist',
+        'id': videos_id,
+        'entries': [{
+            '_type': 'url',
+            'url': "http://www.vier.be/" + eurl[0],
+            'ie_key': 'Vier',
+          } for eurl in re.findall (r'<h3><a href="(.+?)">(.+?)</a></h3>', videos_page)]
+        }

From 9eb4f404cbd10268befa47a1bcf699f07a47f5de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:15:40 +0600
Subject: [PATCH 353/974] [vier] Simplify, add support for more URL formats,
 extract all playlist pages when page is not specified

---
 youtube_dl/extractor/vier.py | 169 ++++++++++++++++++++++-------------
 1 file changed, 106 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index e3f1b4714..fe54ae475 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -1,75 +1,118 @@
-from .common import InfoExtractor
-from ..utils import escape_url
+# coding: utf-8
+from __future__ import unicode_literals
 
 import re
 
-class VierIE (InfoExtractor):
-  _VALID_URL = r'(?:http://)?www.vier.be/(?P<program>.*)/videos/(.+?)/(?P<id>\d*)'
-  _TEST = {
-      'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
-      'md5': 'bf48f4eb998cbde44ecd02fc42c51149',
-      'info_dict': {
-          'id': '16129',
-          'ext': 'mp4',
-          'title': 'Het wordt warm in De Moestuin',
-          'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
-      },
-  }
+from .common import InfoExtractor
 
-  def _real_extract (self, url):
-    mobj = re.match (self._VALID_URL, url)
 
-    program = mobj.group ('program')
-    video_id = mobj.group ('id')
+class VierIE(InfoExtractor):
+    IE_NAME = 'vier'
+    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
+    _TESTS = [{
+        'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
+        'info_dict': {
+            'id': '16129',
+            'display_id': 'het-wordt-warm-de-moestuin',
+            'ext': 'mp4',
+            'title': 'Het wordt warm in De Moestuin',
+            'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.vier.be/planb/videos/mieren-herders-van-de-bladluizen',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.vier.be/video/v3/embed/16129',
+        'only_matching': True,
+    }]
 
-    webpage = self._download_webpage (url, video_id)
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        embed_id = mobj.group('embed_id')
+        display_id = mobj.group('display_id') or embed_id
 
-    title = self._html_search_regex(r'<meta property="og:title" content="(.+?)" />', webpage, u'title')
-    description = self._html_search_regex (r'<meta property="og:description" content="(.+?)" />', webpage, u'description')
-    vod_id =  self._html_search_regex(r'"filename" : "(.+?)"', webpage, u'playlist URL')
-    url = escape_url ("http://vod.streamcloud.be/vier_vod/mp4:_definst_/" + vod_id + ".mp4/playlist.m3u8")
+        webpage = self._download_webpage(url, display_id)
 
-    return {
-        'id': video_id,
-        'title': title,
-        'description': description,
-        'formats': self._extract_m3u8_formats(url, video_id, 'mp4'),
-    }
+        video_id = self._search_regex(
+            r'"nid"\s*:\s*"(\d+)"', webpage, 'video id')
+        application = self._search_regex(
+            r'"application"\s*:\s*"([^"]+)"', webpage, 'application', default='vier_vod')
+        filename = self._search_regex(
+            r'"filename"\s*:\s*"([^"]+)"', webpage, 'filename')
 
-class VierVideosIE (InfoExtractor):
-  _VALID_URL = r'http://www.vier.be/(?P<program>.*)/videos(\?page=(?P<page>\d*))?$'
-  _TESTS = [{
-    'url': 'http://www.vier.be/demoestuin/videos',
-    'info_dict': {
-      'id': 'demoestuin page(0)',
-    },
-    'playlist_mincount': 20,
-  },
-  {
-    'url': 'http://www.vier.be/demoestuin/videos?page=6',
-    'info_dict': {
-      'id': 'demoestuin page(6)',
-    },
-    'playlist_mincount': 20,
-  }]
+        playlist_url = 'http://vod.streamcloud.be/%s/mp4:_definst_/%s.mp4/playlist.m3u8' % (application, filename)
+        formats = self._extract_m3u8_formats(playlist_url, display_id, 'mp4')
 
-  def _real_extract (self, url):
-    mobj = re.match (self._VALID_URL, url)
+        title = self._og_search_title(webpage, default=display_id)
+        description = self._og_search_description(webpage, default=None)
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
 
-    program = mobj.group ('program')
-    page = mobj.group ('page')
-    if page == None:
-      page = 0
-
-    videos_id = program + " page(" + str (page) + ")"
-    videos_page = self._download_webpage (url, videos_id, note='Retrieving videos page')
-
-    return {
-        '_type': 'playlist',
-        'id': videos_id,
-        'entries': [{
-            '_type': 'url',
-            'url': "http://www.vier.be/" + eurl[0],
-            'ie_key': 'Vier',
-          } for eurl in re.findall (r'<h3><a href="(.+?)">(.+?)</a></h3>', videos_page)]
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
         }
+
+
+class VierVideosIE(InfoExtractor):
+    IE_NAME = 'vier:videos'
+    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?page=(?P<page>\d+))?$'
+    _TESTS = [{
+        'url': 'http://www.vier.be/demoestuin/videos',
+        'info_dict': {
+            'id': 'demoestuin',
+        },
+        'playlist_mincount': 153,
+    }, {
+        'url': 'http://www.vier.be/demoestuin/videos?page=6',
+        'info_dict': {
+            'id': 'demoestuin-page6',
+        },
+        'playlist_mincount': 20,
+    }, {
+        'url': 'http://www.vier.be/demoestuin/videos?page=7',
+        'info_dict': {
+            'id': 'demoestuin-page7',
+        },
+        'playlist_mincount': 13,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        program = mobj.group('program')
+
+        webpage = self._download_webpage(url, program)
+
+        page_id = mobj.group('page')
+        if page_id:
+            page_id = int(page_id)
+            start_page = page_id
+            last_page = start_page + 1
+            playlist_id = '%s-page%d' % (program, page_id)
+        else:
+            start_page = 0
+            last_page = int(self._search_regex(
+                r'videos\?page=(\d+)">laatste</a>',
+                webpage, 'last page', default=0)) + 1
+            playlist_id = program
+
+        entries = []
+        for current_page_id in range(start_page, last_page):
+            current_page = self._download_webpage(
+                'http://www.vier.be/%s/videos?page=%d' % (program, current_page_id),
+                program,
+                'Downloading page %d' % (current_page_id + 1)) if current_page_id != page_id else webpage
+            page_entries = [
+                self.url_result('http://www.vier.be' + video_url, 'Vier')
+                for video_url in re.findall(
+                    r'<h3><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', current_page)]
+            entries.extend(page_entries)
+
+        return self.playlist_result(entries, playlist_id)

From 6f790e582197b8f2366fc8fa94f76fe9d83954be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:16:43 +0600
Subject: [PATCH 354/974] Credit @lovebug356 for vier (#4617)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 29ce9e3e4..37306ac83 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -97,3 +97,4 @@ Petr Kutalek
 Will Glynn
 Max Reimann
 Cédric Luthi
+Thijs Vermeir

From fdbabca85f80e47961329972bcbf627f25638ce4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:21:41 +0600
Subject: [PATCH 355/974] [vier:videos] Tune _VALID_URL

---
 youtube_dl/extractor/vier.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index fe54ae475..0d9fb09a7 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -63,7 +63,7 @@ class VierIE(InfoExtractor):
 
 class VierVideosIE(InfoExtractor):
     IE_NAME = 'vier:videos'
-    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?page=(?P<page>\d+))?$'
+    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+))?'
     _TESTS = [{
         'url': 'http://www.vier.be/demoestuin/videos',
         'info_dict': {

From 317639758a0f1728e6e1d612800b6eab69261e89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:37:54 +0600
Subject: [PATCH 356/974] [bbccouk] Improve _VALID_URL

---
 youtube_dl/extractor/bbccouk.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 53aea0ec9..73fe66b01 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -10,7 +10,7 @@ from ..compat import compat_HTTPError
 class BBCCoUkIE(SubtitlesInfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:programmes|iplayer/episode)/(?P<id>[\da-z]{8})'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:programmes|iplayer/(?:episode|playlist))/(?P<id>[\da-z]{8})'
 
     _TESTS = [
         {
@@ -84,6 +84,9 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
                 # rtmp download
                 'skip_download': True,
             }
+        }, {
+            'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
+            'only_matching': True,
         }
     ]
 

From db546cf87fa4206e55f66ddcc041dcd12fe708c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:46:17 +0600
Subject: [PATCH 357/974] [generic] Add support for BBC iPlayer embeds (Closes
 #4619)

---
 youtube_dl/extractor/generic.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 40b2791c7..ce1e77daa 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -905,6 +905,11 @@ class GenericIE(InfoExtractor):
             return _playlist_from_matches(
                 matches, getter=unescapeHTML, ie='FunnyOrDie')
 
+        # Look for BBC iPlayer embed
+        matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
+        if matches:
+            return self.playlist_result([self.url_result(video_url, ie='BBCCoUk') for video_url in matches])
+
         # Look for embedded RUTV player
         rutv_url = RUTVIE._extract_url(webpage)
         if rutv_url:

From 8399267671938f072b9c6f6b2cbd2fa315b80a23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:54:30 +0600
Subject: [PATCH 358/974] [generic] Make getter None by default

---
 youtube_dl/extractor/generic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ce1e77daa..4bb7581ac 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -699,9 +699,9 @@ class GenericIE(InfoExtractor):
             r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
 
         # Helper method
-        def _playlist_from_matches(matches, getter, ie=None):
+        def _playlist_from_matches(matches, getter=None, ie=None):
             urlrs = orderedSet(
-                self.url_result(self._proto_relative_url(getter(m)), ie)
+                self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
                 for m in matches)
             return self.playlist_result(
                 urlrs, playlist_id=video_id, playlist_title=video_title)

From 476eae0c2a7cb4de1f00d050d733f1df1bd6cc0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:55:09 +0600
Subject: [PATCH 359/974] [generic] Generalize BBC iPlayer playlist extraction

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4bb7581ac..fd4ee4667 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -908,7 +908,7 @@ class GenericIE(InfoExtractor):
         # Look for BBC iPlayer embed
         matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
         if matches:
-            return self.playlist_result([self.url_result(video_url, ie='BBCCoUk') for video_url in matches])
+            return _playlist_from_matches(matches, ie='BBCCoUk')
 
         # Look for embedded RUTV player
         rutv_url = RUTVIE._extract_url(webpage)

From 29a9385ff083fde37bcd5fc1119fa7fa55cf6973 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 2 Jan 2015 15:56:26 +0100
Subject: [PATCH 360/974] release 2015.01.02

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 89707ceef..58b5021dc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.01'
+__version__ = '2015.01.02'

From faa4ea68c08efc5bddb863455bab7c2fe64dcbc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:56:42 +0600
Subject: [PATCH 361/974] [generic] Add BBC iPlayer playlist test

---
 youtube_dl/extractor/generic.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index fd4ee4667..493afb57d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -181,6 +181,14 @@ class GenericIE(InfoExtractor):
                 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
             },
         },
+        # BBC iPlayer embeds
+        {
+            'url': 'http://www.bbc.co.uk/blogs/adamcurtis/posts/BUGGER',
+            'info_dict': {
+                'title': 'BBC - Blogs -  Adam Curtis - BUGGER',
+            },
+            'playlist_mincount': 18,
+        },
         # RUTV embed
         {
             'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',

From d34f98289b302619b1213ef6e50e034021f811e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 21:12:25 +0600
Subject: [PATCH 362/974] [xhamster] Remove identical tests

---
 youtube_dl/extractor/xhamster.py | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 519bfc191..4527567f8 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -42,28 +42,8 @@ class XHamsterIE(InfoExtractor):
         },
         {
             'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',
-            'info_dict': {
-                'id': '2272726',
-                'ext': 'mp4',
-                'title': 'Amber slayed by the Knight',
-                'upload_date': '20131009',
-                'uploader_id': 'amberblank',
-                'duration': 149,
-                'age_limit': 18,
-            }
+            'only_matching': True,
         },
-        {
-            "url": "https://xhamster.com/movies/1444747/hotkinkyjo_amp_asian_girl_elbow_deep_anal_fisting_and_footing.html",
-            'info_dict': {
-                "id": "1444747",
-                'ext': 'mp4',
-                "title": "HOTKINKYJO & ASIAN GIRL ELBOW DEEP ANAL FISTING AND FOOTING",
-                "upload_date": "20120922",
-                "uploader_id": "alex1981",
-                "duration": 80,
-                "age_limit": 18,
-            }
-        }
     ]
 
     def _real_extract(self, url):

From 324b2c78fad949545b6e0707bb5ee1e37ec6217d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 21:46:57 +0600
Subject: [PATCH 363/974] [xtube] Fix uploader regex

---
 youtube_dl/extractor/xtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index cf74d4fd5..95f1c8f3c 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -40,7 +40,7 @@ class XTubeIE(InfoExtractor):
             r'<p class="title">([^<]+)', webpage, 'title')
         video_uploader = self._html_search_regex(
             [r"var\s+contentOwnerId\s*=\s*'([^']+)",
-             r'By:\s*<a href="/community/profile\.php?user=([^"]+)'],
+             r'By:\s*<a href="/community/profile\.php\?user=([^"]+)'],
             webpage, 'uploader', fatal=False)
         video_description = self._html_search_regex(
             r'<p class="fieldsDesc">([^<]+)',

From 823a155293f744f147c83c2f8efc4aaf2c3760e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 22:09:00 +0600
Subject: [PATCH 364/974] [vier:videos] Tune _VALID_URL not to match single
 videos

---
 youtube_dl/extractor/vier.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 0d9fb09a7..619039e51 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -63,7 +63,7 @@ class VierIE(InfoExtractor):
 
 class VierVideosIE(InfoExtractor):
     IE_NAME = 'vier:videos'
-    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+))?'
+    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+)|$)'
     _TESTS = [{
         'url': 'http://www.vier.be/demoestuin/videos',
         'info_dict': {

From c067545c175bb421fcfc255ae7df7a2fff1c5638 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= <ondrej@caletka.cz>
Date: Fri, 2 Jan 2015 17:12:20 +0100
Subject: [PATCH 365/974] ceskatelevize: Closed captions support

---
 youtube_dl/extractor/ceskatelevize.py | 55 ++++++++++++++++++++++++++-
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index ba8376338..345ac5e85 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
@@ -12,10 +12,11 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     float_or_none,
+    HEADRequest,
 )
 
 
-class CeskaTelevizeIE(InfoExtractor):
+class CeskaTelevizeIE(SubtitlesInfoExtractor):
     _VALID_URL = r'https?://www\.ceskatelevize\.cz/(porady|ivysilani)/(.+/)?(?P<id>[^?#]+)'
 
     _TESTS = [
@@ -104,6 +105,15 @@ class CeskaTelevizeIE(InfoExtractor):
         duration = float_or_none(item.get('duration'))
         thumbnail = item.get('previewImageUrl')
 
+        # subtitles
+        subtitles = self.extract_subtitles(video_id, webpage)
+
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, webpage)
+            return
+
+        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, webpage))
+
         return {
             'id': episode_id,
             'title': title,
@@ -111,4 +121,45 @@ class CeskaTelevizeIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
+            'subtitles': subtitles,
         }
+
+    def _fix_subtitles(self, subtitles):
+        """ Convert milisecond-based subtitles to SRT """
+        if subtitles is None:
+            return subtitles  # subtitles not requested
+
+        def _msectotimecode(msec):
+            """ Helper utility to convert miliseconds to timecode """
+            components = []
+            for divider in [1000, 60, 60, 100]:
+                components.append(msec % divider)
+                msec //= divider
+            return "{3:02}:{2:02}:{1:02},{0:03}".format(*components)
+
+        def _fix_subtitle(subtitle):
+            for line in subtitle.splitlines():
+                m = re.match(r"^ *([0-9]+); *([0-9]+) +([0-9]+) *$", line)
+                if m:
+                    yield m.group(1)
+                    start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
+                    yield "{} --> {}".format(start, stop)
+                else:
+                    yield line
+
+        fixed_subtitles = {}
+        for k, v in subtitles.items():
+            fixed_subtitles[k] = "\r\n".join(_fix_subtitle(v))
+        return fixed_subtitles
+
+    def _get_available_subtitles(self, video_id, webpage):
+        video_id = video_id.partition('-')[0]
+        url = 'http://imgct.ceskatelevize.cz/cache/data/ivysilani/' \
+              'subtitles/{}/{}/sub.txt'.format(video_id[:3], video_id)
+        req = HEADRequest(url)
+        sub = self._request_webpage(
+            req, video_id,
+            note="Checking subtitles",
+            errnote="No subtitles found",
+            fatal=False)
+        return {'cs': url} if sub else {}

From c491418526dc87d16188e30d6bf1fc01917ad62c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 22:13:26 +0600
Subject: [PATCH 366/974] [bbccouk] Update test

---
 youtube_dl/extractor/bbccouk.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 73fe66b01..5865a6e39 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -18,8 +18,8 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             'info_dict': {
                 'id': 'b039d07m',
                 'ext': 'flv',
-                'title': 'Kaleidoscope: Leonard Cohen',
-                'description': 'md5:db4755d7a665ae72343779f7dacb402c',
+                'title': 'Kaleidoscope, Leonard Cohen',
+                'description': 'The Canadian poet and songwriter reflects on his musical career.',
                 'duration': 1740,
             },
             'params': {

From 46d32a12c9fd3e7639a51e27226a29c59ee0ca72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 22:23:00 +0600
Subject: [PATCH 367/974] [bet] Update test

---
 youtube_dl/extractor/bet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index 003e50002..4fbdd6f1c 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -16,7 +16,7 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
             'info_dict': {
-                'id': '417cd61c-c793-4e8e-b006-e445ecc45add',
+                'id': '406429c6-1b8a-463e-83fc-814adb81a9db',
                 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
                 'ext': 'flv',
                 'title': 'BET News Presents: A Conversation With President Obama',

From 2522a0b7dab3acde63ed1384d53a6e5c2b7fa749 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 22:28:48 +0600
Subject: [PATCH 368/974] [kontrtube] Extract display_id

Trailing slash in URL is mandatory now
---
 youtube_dl/extractor/kontrtube.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/kontrtube.py b/youtube_dl/extractor/kontrtube.py
index 41fd62009..720bc939b 100644
--- a/youtube_dl/extractor/kontrtube.py
+++ b/youtube_dl/extractor/kontrtube.py
@@ -10,13 +10,14 @@ from ..utils import int_or_none
 class KontrTubeIE(InfoExtractor):
     IE_NAME = 'kontrtube'
     IE_DESC = 'KontrTube.ru - Труба зовёт'
-    _VALID_URL = r'http://(?:www\.)?kontrtube\.ru/videos/(?P<id>\d+)/.+'
+    _VALID_URL = r'http://(?:www\.)?kontrtube\.ru/videos/(?P<id>\d+)/(?P<display_id>[^/]+)/'
 
     _TEST = {
         'url': 'http://www.kontrtube.ru/videos/2678/nad-olimpiyskoy-derevney-v-sochi-podnyat-rossiyskiy-flag/',
         'md5': '975a991a4926c9a85f383a736a2e6b80',
         'info_dict': {
             'id': '2678',
+            'display_id': 'nad-olimpiyskoy-derevney-v-sochi-podnyat-rossiyskiy-flag',
             'ext': 'mp4',
             'title': 'Над олимпийской деревней в Сочи поднят российский флаг',
             'description': 'md5:80edc4c613d5887ae8ccf1d59432be41',
@@ -28,21 +29,28 @@ class KontrTubeIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, video_id, 'Downloading page')
+        webpage = self._download_webpage(
+            url, display_id, 'Downloading page')
 
-        video_url = self._html_search_regex(r"video_url: '(.+?)/?',", webpage, 'video URL')
-        thumbnail = self._html_search_regex(r"preview_url: '(.+?)/?',", webpage, 'video thumbnail', fatal=False)
+        video_url = self._html_search_regex(
+            r"video_url\s*:\s*'(.+?)/?',", webpage, 'video URL')
+        thumbnail = self._html_search_regex(
+            r"preview_url\s*:\s*'(.+?)/?',", webpage, 'video thumbnail', fatal=False)
         title = self._html_search_regex(
             r'<title>(.+?)</title>', webpage, 'video title')
-        description = self._html_search_meta('description', webpage, 'video description')
+        description = self._html_search_meta(
+            'description', webpage, 'video description')
 
         mobj = re.search(
-            r'<div class="col_2">Длительность: <span>(?P<minutes>\d+)м:(?P<seconds>\d+)с</span></div>', webpage)
+            r'<div class="col_2">Длительность: <span>(?P<minutes>\d+)м:(?P<seconds>\d+)с</span></div>',
+            webpage)
         duration = int(mobj.group('minutes')) * 60 + int(mobj.group('seconds')) if mobj else None
 
         view_count = self._html_search_regex(
-            r'<div class="col_2">Просмотров: <span>(\d+)</span></div>', webpage, 'view count', fatal=False)
+            r'<div class="col_2">Просмотров: <span>(\d+)</span></div>',
+            webpage, 'view count', fatal=False)
 
         comment_count = None
         comment_str = self._html_search_regex(
@@ -56,6 +64,7 @@ class KontrTubeIE(InfoExtractor):
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'url': video_url,
             'thumbnail': thumbnail,
             'title': title,

From 62420c73cb16472ead562339e22b038fac7aa950 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 22:31:55 +0600
Subject: [PATCH 369/974] [played] Skip test

---
 youtube_dl/extractor/played.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 449d4836c..45716c75d 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -26,6 +26,7 @@ class PlayedIE(InfoExtractor):
             'ext': 'flv',
             'title': 'youtube-dl_test_video.mp4',
         },
+        'skip': 'Removed for copyright infringement.',  # oh wow
     }
 
     def _real_extract(self, url):

From 817f786fbb05070a7ad23aec32293cfcdd072bf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Jan 2015 21:02:34 +0100
Subject: [PATCH 370/974] [canalplus] Raise an error if the video is
 georestricted (closes #4472)

---
 youtube_dl/extractor/canalplus.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 9873728df..11d18d74a 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -5,6 +5,8 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
+    HEADRequest,
     unified_strdate,
     url_basename,
     qualities,
@@ -76,6 +78,16 @@ class CanalplusIE(InfoExtractor):
 
         preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS'])
 
+        fmt_url = next(iter(media.find('VIDEOS'))).text
+        if '/geo' in fmt_url.lower():
+            response = self._request_webpage(
+                HEADRequest(fmt_url), video_id,
+                'Checking if the video is georestricted')
+            if '/blocage' in response.geturl():
+                raise ExtractorError(
+                    'The video is not available in your country',
+                    expected=True)
+
         formats = []
         for fmt in media.find('VIDEOS'):
             format_url = fmt.text

From a221f22969843866ad85fd5cdf98160c79d99e9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Jan 2015 21:17:10 +0100
Subject: [PATCH 371/974] [crunchyroll] Fix format extraction

Reported in https://github.com/rg3/youtube-dl/issues/2782#issuecomment-68556780
---
 youtube_dl/extractor/crunchyroll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 354046a9e..1680f532f 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -228,7 +228,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False)
 
         formats = []
-        for fmt in re.findall(r'\?p([0-9]{3,4})=1', webpage):
+        for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
             streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/')

From d26b1317edd31d85f8fa68d796f185f7ba65c282 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 3 Jan 2015 00:33:36 +0100
Subject: [PATCH 372/974] [downloader/mplayer] Use check_executable

---
 youtube_dl/downloader/mplayer.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py
index c53195da0..34b23b5c2 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/mplayer.py
@@ -6,6 +6,7 @@ import subprocess
 from .common import FileDownloader
 from ..compat import compat_subprocess_get_DEVNULL
 from ..utils import (
+    check_executable,
     encodeFilename,
 )
 
@@ -20,11 +21,7 @@ class MplayerFD(FileDownloader):
             'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
             '-dumpstream', '-dumpfile', tmpfilename, url]
         # Check for mplayer first
-        try:
-            subprocess.call(
-                ['mplayer', '-h'],
-                stdout=compat_subprocess_get_DEVNULL(), stderr=subprocess.STDOUT)
-        except (OSError, IOError):
+        if not check_executable('mplayer', ['-h']):
             self.report_error('MMS or RTSP download detected but "%s" could not be run' % args[0])
             return False
 

From c7e675940ca06a74f068164b25d3afb9b657ff24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Jan 2015 20:43:40 +0600
Subject: [PATCH 373/974] [bbccouk] Add support for music clips (Closes #4143)

---
 youtube_dl/extractor/bbccouk.py | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 5865a6e39..1cf48fe0d 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -10,7 +10,7 @@ from ..compat import compat_HTTPError
 class BBCCoUkIE(SubtitlesInfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:programmes|iplayer/(?:episode|playlist))/(?P<id>[\da-z]{8})'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
     _TESTS = [
         {
@@ -84,9 +84,40 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
                 # rtmp download
                 'skip_download': True,
             }
+        }, {
+            'url': 'http://www.bbc.co.uk/music/clips/p02frcc3',
+            'note': 'Audio',
+            'info_dict': {
+                'id': 'p02frcch',
+                'ext': 'flv',
+                'title': 'Pete Tong, Past, Present and Future Special, Madeon - After Hours mix',
+                'description': 'French house superstar Madeon takes us out of the club and onto the after party.',
+                'duration': 3507,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            }
+        }, {
+            'url': 'http://www.bbc.co.uk/music/clips/p025c0zz',
+            'note': 'Video',
+            'info_dict': {
+                'id': 'p025c103',
+                'ext': 'flv',
+                'title': 'Reading and Leeds Festival, 2014, Rae Morris - Closer (Live on BBC Three)',
+                'description': 'Rae Morris performs Closer for BBC Three at Reading 2014',
+                'duration': 226,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            }
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,
+        }, {
+            'url': 'http://www.bbc.co.uk/music/clips#p02frcc3',
+            'only_matching': True,
         }
     ]
 

From 6e46c3f1fddc954596d429d15d7c035d12a625a9 Mon Sep 17 00:00:00 2001
From: Gaurav <gauravb7090@gmail.com>
Date: Sat, 3 Jan 2015 20:30:28 +0530
Subject: [PATCH 374/974] Added support for EllenTube along with EllenTV

---
 youtube_dl/extractor/ellentv.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 3e7923648..097317842 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -12,8 +12,8 @@ from ..utils import (
 
 
 class EllenTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ellentv\.com/videos/(?P<id>[a-z0-9_-]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
+    _TESTS = [{
         'url': 'http://www.ellentv.com/videos/0-7jqrsr18/',
         'md5': 'e4af06f3bf0d5f471921a18db5764642',
         'info_dict': {
@@ -23,7 +23,18 @@ class EllenTVIE(InfoExtractor):
             'timestamp': 1406876400,
             'upload_date': '20140801',
         }
-    }
+    },
+    {
+        'url': 'http://ellentube.com/videos/0-dvzmabd5/',
+        'md5': '98238118eaa2bbdf6ad7f708e3e4f4eb',
+        'info_dict': {
+            'id': '0-dvzmabd5',
+            'ext': 'mp4',
+            'title': '1 year old twin sister makes her brother laugh',
+            'timestamp': 1419542075,
+            'upload_date': '20141225',
+        }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -33,6 +44,9 @@ class EllenTVIE(InfoExtractor):
         timestamp = parse_iso8601(self._search_regex(
             r'<span class="publish-date"><time datetime="([^"]+)">',
             webpage, 'timestamp'))
+        print "\n"
+        print timestamp
+        print "\n"
 
         return {
             'id': video_id,

From f4f339529cfe19d09a0c7103f64ef60b15032a2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Jan 2015 21:44:47 +0600
Subject: [PATCH 375/974] [ellentv] Clean up and simplify

---
 youtube_dl/extractor/ellentv.py | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 097317842..89fa9f7e4 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import json
 
 from .common import InfoExtractor
@@ -12,7 +11,7 @@ from ..utils import (
 
 
 class EllenTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
     _TESTS = [{
         'url': 'http://www.ellentv.com/videos/0-7jqrsr18/',
         'md5': 'e4af06f3bf0d5f471921a18db5764642',
@@ -23,8 +22,7 @@ class EllenTVIE(InfoExtractor):
             'timestamp': 1406876400,
             'upload_date': '20140801',
         }
-    },
-    {
+    }, {
         'url': 'http://ellentube.com/videos/0-dvzmabd5/',
         'md5': '98238118eaa2bbdf6ad7f708e3e4f4eb',
         'info_dict': {
@@ -37,16 +35,12 @@ class EllenTVIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         timestamp = parse_iso8601(self._search_regex(
             r'<span class="publish-date"><time datetime="([^"]+)">',
             webpage, 'timestamp'))
-        print "\n"
-        print timestamp
-        print "\n"
 
         return {
             'id': video_id,
@@ -69,8 +63,7 @@ class EllenTVClipsIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
+        playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
         playlist = self._extract_playlist(webpage)

From baa3c3f0f64b1aba523f56c9af351b6fb3378fc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Jan 2015 21:54:18 +0600
Subject: [PATCH 376/974] [ellentv] Improve extraction

---
 youtube_dl/extractor/ellentv.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 89fa9f7e4..fc92ff825 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -19,6 +19,7 @@ class EllenTVIE(InfoExtractor):
             'id': '0-7jqrsr18',
             'ext': 'mp4',
             'title': 'What\'s Wrong with These Photos? A Whole Lot',
+            'description': 'md5:35f152dc66b587cf13e6d2cf4fa467f6',
             'timestamp': 1406876400,
             'upload_date': '20140801',
         }
@@ -29,6 +30,7 @@ class EllenTVIE(InfoExtractor):
             'id': '0-dvzmabd5',
             'ext': 'mp4',
             'title': '1 year old twin sister makes her brother laugh',
+            'description': '1 year old twin sister makes her brother laugh',
             'timestamp': 1419542075,
             'upload_date': '20141225',
         }
@@ -38,14 +40,20 @@ class EllenTVIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
+        video_url = self._html_search_meta('VideoURL', webpage, 'url')
+        title = self._og_search_title(webpage, default=None) or self._search_regex(
+            r'pageName\s*=\s*"([^"]+)"', webpage, 'title')
+        description = self._html_search_meta(
+            'description', webpage, 'description') or self._og_search_description(webpage)
         timestamp = parse_iso8601(self._search_regex(
             r'<span class="publish-date"><time datetime="([^"]+)">',
             webpage, 'timestamp'))
 
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage),
-            'url': self._html_search_meta('VideoURL', webpage, 'url'),
+            'url': video_url,
+            'title': title,
+            'description': description,
             'timestamp': timestamp,
         }
 

From e65566a9cc13d13bef3dc70eebfc9bda2e302b78 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 3 Jan 2015 18:33:38 +0100
Subject: [PATCH 377/974] [youtube] Correct handling when DASH manifest is not
 necessary to find all formats

---
 youtube_dl/YoutubeDL.py         |  4 +++-
 youtube_dl/extractor/common.py  |  2 ++
 youtube_dl/extractor/youtube.py | 11 +++++++++--
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e2b823f66..806e7b239 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1333,7 +1333,9 @@ class YoutubeDL(object):
         formats = info_dict.get('formats', [info_dict])
         idlen = max(len('format code'),
                     max(len(f['format_id']) for f in formats))
-        formats_s = [line(f, idlen) for f in formats]
+        formats_s = [
+            line(f, idlen) for f in formats
+            if f.get('preference') is None or f['preference'] >= -1000]
         if len(formats) > 1:
             formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)'
             formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6e264f687..562e656e0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -92,6 +92,8 @@ class InfoExtractor(object):
                                  by this field, regardless of all other values.
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
+                                 < -1000 to hide the format (if there is
+                                    another one which is strictly better)
                     * language_preference  Is this in the correct requested
                                  language?
                                  10 if it's what the URL is about,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8c7842ee8..e9bf39a00 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -256,7 +256,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '138': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
@@ -736,6 +736,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'format_id': format_id,
                 'url': video_url,
                 'width': int_or_none(r.attrib.get('width')),
+                'height': int_or_none(r.attrib.get('height')),
                 'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
                 'asr': int_or_none(r.attrib.get('audioSamplingRate')),
                 'filesize': filesize,
@@ -746,7 +747,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     fo for fo in formats
                     if fo['format_id'] == format_id)
             except StopIteration:
-                f.update(self._formats.get(format_id, {}))
+                f.update(self._formats.get(format_id, {}).items())
                 formats.append(f)
             else:
                 existing_format.update(f)
@@ -1040,6 +1041,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)
                 else:
+                    # Hide the formats we found through non-DASH
+                    dash_keys = set(df['format_id'] for df in dash_formats)
+                    for f in formats:
+                        if f['format_id'] in dash_keys:
+                            f['format_id'] = 'nondash-%s' % f['format_id']
+                            f['preference'] -= 10000
                     formats.extend(dash_formats)
 
         self._sort_formats(formats)

From 19b05d886ef0bd23f4a4c6ee90f171c48cb55ace Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 3 Jan 2015 18:35:30 +0100
Subject: [PATCH 378/974] release 2015.01.03

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 58b5021dc..2a54b9bbe 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.02'
+__version__ = '2015.01.03'

From 382e05fa5686ea58606f684f3032b91801ece8f3 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Sun, 4 Jan 2015 00:05:31 +0000
Subject: [PATCH 379/974] [tf1] add support for TFOU

---
 youtube_dl/extractor/tf1.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 6e61cc9e2..384b4c3f8 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -8,8 +8,8 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://videos\.tf1\.fr/.*-(?P<id>.*?)\.html'
-    _TEST = {
+    _VALID_URL = r'http://(?:videos\.tf1|www\.tfou)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
+    _TESTS = {
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
             'id': '10635995',
@@ -21,14 +21,25 @@ class TF1IE(InfoExtractor):
             # Sometimes wat serves the whole file with the --test option
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.tfou.fr/chuggington/videos/le-grand-mysterioso-chuggington-7085291-739.html',
+        'info_dict': {
+            'id': '7085291',
+            'ext': 'mp4',
+            'title': 'Le grand Mystérioso - Chuggington',
+            'description': 'Emery rêve qu\'un article lui soit consacré dans le journal.',
+        },
+        'params': {
+            # Sometimes wat serves the whole file with the --test option
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         embed_url = self._html_search_regex(
-            r'"(https://www.wat.tv/embedframe/.*?)"', webpage, 'embed url')
+            r'["\'](http(?:s)?://www.wat.tv/embedframe/.*?)["\']', webpage, 'embed url')
         embed_page = self._download_webpage(embed_url, video_id,
                                             'Downloading embed player page')
         wat_id = self._search_regex(r'UVID=(.*?)&', embed_page, 'wat id')

From 2bf098eda43f0c789e04f7f633149cc99873d182 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Sun, 4 Jan 2015 00:43:55 +0000
Subject: [PATCH 380/974] [tf1] fix test

---
 youtube_dl/extractor/tf1.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 384b4c3f8..5c0fcef58 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -24,10 +24,11 @@ class TF1IE(InfoExtractor):
     }, {
         'url': 'http://www.tfou.fr/chuggington/videos/le-grand-mysterioso-chuggington-7085291-739.html',
         'info_dict': {
-            'id': '7085291',
+            'id': '12043945',
             'ext': 'mp4',
             'title': 'Le grand Mystérioso - Chuggington',
-            'description': 'Emery rêve qu\'un article lui soit consacré dans le journal.',
+            'description': 'Le grand Mystérioso - Emery rêve qu\'un article lui soit consacré dans le journal.',
+            'upload_date': '20150103',
         },
         'params': {
             # Sometimes wat serves the whole file with the --test option

From b158bb86934fb8863a697b6e65a1cbc5105d1741 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Sun, 4 Jan 2015 00:45:23 +0000
Subject: [PATCH 381/974] [tf1] simplify regex

---
 youtube_dl/extractor/tf1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 5c0fcef58..07cc81226 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -40,7 +40,7 @@ class TF1IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         embed_url = self._html_search_regex(
-            r'["\'](http(?:s)?://www.wat.tv/embedframe/.*?)["\']', webpage, 'embed url')
+            r'["\'](https?://www.wat.tv/embedframe/.*?)["\']', webpage, 'embed url')
         embed_page = self._download_webpage(embed_url, video_id,
                                             'Downloading embed player page')
         wat_id = self._search_regex(r'UVID=(.*?)&', embed_page, 'wat id')

From b68ff259170711574a1fd779e86bdfea9aaafcf5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 02:05:26 +0100
Subject: [PATCH 382/974] Add various anime sites (Closes #4554)

---
 AUTHORS                           |   1 +
 youtube_dl/extractor/__init__.py  |  19 ++++
 youtube_dl/extractor/gogoanime.py |  76 +++++++++++++++
 youtube_dl/extractor/play44.py    | 149 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/soulanime.py |  74 +++++++++++++++
 youtube_dl/extractor/videofun.py  |  36 ++++++++
 6 files changed, 355 insertions(+)
 create mode 100644 youtube_dl/extractor/gogoanime.py
 create mode 100644 youtube_dl/extractor/play44.py
 create mode 100644 youtube_dl/extractor/soulanime.py
 create mode 100644 youtube_dl/extractor/videofun.py

diff --git a/AUTHORS b/AUTHORS
index 37306ac83..9b548cf25 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -98,3 +98,4 @@ Will Glynn
 Max Reimann
 Cédric Luthi
 Thijs Vermeir
+Joel Leclerc
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9ccd1b32e..c3dc09f75 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -164,6 +164,10 @@ from .globo import GloboIE
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
+from .gogoanime import (
+    GoGoAnimeIE,
+    GoGoAnimeSearchIE
+)
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
@@ -313,6 +317,16 @@ from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .planetaplay import PlanetaPlayIE
 from .played import PlayedIE
+from .play44 import (
+    Play44IE,
+    ByZooIE,
+    Video44IE,
+    VideoWingIE,
+    PlayPandaIE,
+    VideoZooIE,
+    PlayBBIE,
+    EasyVideoIE
+)
 from .playfm import PlayFMIE
 from .playvid import PlayvidIE
 from .podomatic import PodomaticIE
@@ -373,6 +387,10 @@ from .smotri import (
 from .snotr import SnotrIE
 from .sockshare import SockshareIE
 from .sohu import SohuIE
+from .soulanime import (
+    SoulAnimeWatchingIE,
+    SoulAnimeSeriesIE
+)
 from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
@@ -467,6 +485,7 @@ from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
 from .videolecturesnet import VideoLecturesNetIE
+from .videofun import VideoFunIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
 from .videopremium import VideoPremiumIE
diff --git a/youtube_dl/extractor/gogoanime.py b/youtube_dl/extractor/gogoanime.py
new file mode 100644
index 000000000..d4f4ecc58
--- /dev/null
+++ b/youtube_dl/extractor/gogoanime.py
@@ -0,0 +1,76 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    compat_urllib_parse,
+    get_element_by_attribute,
+    unescapeHTML
+)
+
+
+class GoGoAnimeIE(InfoExtractor):
+    IE_NAME = 'gogoanime'
+    IE_DESC = 'GoGoAnime'
+    _VALID_URL = r'http://www.gogoanime.com/(?P<id>[A-Za-z0-9-]+)'
+
+    _TEST = {
+        'url': 'http://www.gogoanime.com/mahou-shoujo-madoka-magica-movie-1',
+        'info_dict': {
+            'id': 'mahou-shoujo-madoka-magica-movie-1'
+        },
+        'playlist_count': 3
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        page = self._download_webpage(url, video_id)
+
+        if 'Oops! Page Not Found</font>' in page:
+            raise ExtractorError('Video does not exist', expected=True)
+
+        content = get_element_by_attribute("class", "postcontent", page)
+        vids = re.findall(r'<iframe[^>]*?src=[\'"](h[^\'"]+)[\'"]', content)
+        vids = [
+            unescapeHTML(compat_urllib_parse.unquote(x))
+            for x in vids if not re.search(r".*videofun.*", x)]
+
+        if re.search(r'<div class="postcontent">[^<]*<p><iframe src=[\'"][^>]+></iframe><br />', page):
+            return self.playlist_result([self.url_result(vid) for vid in vids], video_id)
+
+        title = self._html_search_regex(
+            r'<div class="postdesc">[^<]*<h1>([^<]+)</h1>', page, 'title')
+
+        return {
+            '_type': 'url',
+            'id': video_id,
+            'url': vids[0],
+            'title': title,
+        }
+
+
+class GoGoAnimeSearchIE(InfoExtractor):
+    IE_NAME = 'gogoanime:search'
+    IE_DESC = 'GoGoAnime Search'
+
+    _VALID_URL = r'http://www\.gogoanime\.com/.*\?s=(?P<id>[^&]*)'
+    _TEST = {
+        'url': 'http://www.gogoanime.com/?s=bokusatsu',
+        'info_dict': {
+            'id': 'bokusatsu'
+        },
+        'playlist_count': 6
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+
+        posts = re.findall(
+            r'<div class="postlist">[^<]*<p[^>]*>[^<]*<a href="(?P<url>[^"]+)"',
+            webpage)
+
+        return self.playlist_result(
+            [self.url_result(p) for p in posts], playlist_id)
diff --git a/youtube_dl/extractor/play44.py b/youtube_dl/extractor/play44.py
new file mode 100644
index 000000000..b8696e516
--- /dev/null
+++ b/youtube_dl/extractor/play44.py
@@ -0,0 +1,149 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urllib_parse
+)
+
+
+class Play44IE(InfoExtractor):
+    _VALID_URL = r'http://[w.]*play44\.net/embed\.php[^/]*/(?P<id>.+)'
+
+    _TESTS = [{
+        'url': 'http://play44.net/embed.php?w=600&h=438&vid=M/mahou-shoujo-madoka-magica-07.flv',
+        'md5': 'e37e99d665f503dd2db952f7c4dba9e6',
+        'info_dict': {
+            'id': 'mahou-shoujo-madoka-magica-07',
+            'ext': 'flv',
+            'title': 'mahou-shoujo-madoka-magica-07',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        page = self._download_webpage(url, video_id)
+
+        video_url = compat_urllib_parse.unquote(self._html_search_regex(
+            r'_url = "(https?://[^"]+?)";', page, 'url'))
+        title = self._search_regex(r'.*/(?P<title>[^.]*).', video_url, 'title')
+
+        return {
+            'id': title,
+            'url': video_url,
+            'title': title,
+        }
+
+
+class ByZooIE(Play44IE):
+    _VALID_URL = r'http://[w.]*byzoo\.org/embed\.php[^/]*/(?P<id>.+)'
+
+    _TESTS = [{
+        'url': 'http://byzoo.org/embed.php?w=600&h=438&vid=at/nw/mahou_shoujo_madoka_magica_movie_3_-_part1.mp4',
+        'md5': '455c83dabe2cd9fd74a87612b01fe017',
+        'info_dict': {
+            'id': 'mahou_shoujo_madoka_magica_movie_3_-_part1',
+            'ext': 'mp4',
+            'title': 'mahou_shoujo_madoka_magica_movie_3_-_part1',
+        }
+    }]
+
+
+class Video44IE(Play44IE):
+    _VALID_URL = r'http://[w.]*video44\.net/.*file=(?P<id>[^&].).*'
+
+    _TESTS = [{
+        'url': 'http://www.video44.net/gogo/?w=600&h=438&file=chaoshead-12.flv&sv=1',
+        'md5': '43eaec6d0beb10e8d42459b9f108aff3',
+        'info_dict': {
+            'id': 'chaoshead-12',
+            'ext': 'mp4',
+            'title': 'chaoshead-12',
+        }
+    }]
+
+
+class VideoWingIE(Play44IE):
+    _VALID_URL = r'''(?x)
+        http://[w.]*videowing\.[^/]*/
+        (?:
+            .*video=/*
+            |embed/
+        )
+        (?P<id>[^&?.]+)
+    '''
+
+    _TESTS = [{
+        'url': 'http://videowing.me/embed?w=718&h=438&video=ongoing/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
+        'md5': '4ed320e353ed26c742c4f12a9c210b60',
+        'info_dict': {
+            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
+            'ext': 'mp4',
+            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
+        }
+    }, {
+        'url': 'http://videowing.me/embed/a8d6a39522df066bd734a69f2334497e?w=600&h=438',
+        'md5': '33fdd71581357018c226f95c5cedcfd7',
+        'info_dict': {
+            'id': 'mahoushoujomadokamagicamovie1part1',
+            'ext': 'flv',
+            'title': 'mahoushoujomadokamagicamovie1part1',
+        }
+    }]
+
+
+class PlayPandaIE(Play44IE):
+    _VALID_URL = r'http://[w.]*playpanda\.[^/]*/.*vid=/*(?P<id>[^&].).*'
+
+    _TESTS = [{
+        'url': 'http://playpanda.net/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
+        'md5': '4ed320e353ed26c742c4f12a9c210b60',
+        'info_dict': {
+            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
+            'ext': 'mp4',
+            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
+            'description': 'boku_wa_tomodachi_ga_sukunai_-_05'
+        }
+    }]
+
+
+class VideoZooIE(Play44IE):
+    _VALID_URL = r'http://[w.]*videozoo\.[^/]*/.*vid=/*(?P<id>[^&].).*'
+
+    _TESTS = [{
+        'url': 'http://videozoo.me/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
+        'md5': '4ed320e353ed26c742c4f12a9c210b60',
+        'info_dict': {
+            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
+            'ext': 'mp4',
+            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
+        }
+    }]
+
+
+class PlayBBIE(Play44IE):
+    _VALID_URL = r'http://[w.]*playbb\.[^/]*/.*vid=/*(?P<id>[^&].).*'
+
+    _TESTS = [{
+        'url': 'http://playbb.me/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
+        'md5': '4ed320e353ed26c742c4f12a9c210b60',
+        'info_dict': {
+            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
+            'ext': 'mp4',
+            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
+        }
+    }]
+
+
+class EasyVideoIE(Play44IE):
+    _VALID_URL = r'http://[w.]*easyvideo\.[^/]*/.*file=/*(?P<id>[^&.]+)'
+
+    _TESTS = [{
+        'url': 'http://easyvideo.me/gogo/?w=718&h=438&file=bokuwatomodachigasukunai-04.flv&sv=1',
+        'md5': '26178b57629b7650106d72b191137176',
+        'info_dict': {
+            'id': 'bokuwatomodachigasukunai-04',
+            'ext': 'mp4',
+            'title': 'bokuwatomodachigasukunai-04',
+        },
+        'skip': 'Blocked in Germany',
+    }]
diff --git a/youtube_dl/extractor/soulanime.py b/youtube_dl/extractor/soulanime.py
new file mode 100644
index 000000000..7adb10c03
--- /dev/null
+++ b/youtube_dl/extractor/soulanime.py
@@ -0,0 +1,74 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class SoulAnimeWatchingIE(InfoExtractor):
+    IE_NAME = "soulanime:watching"
+    IE_DESC = "SoulAnime video"
+    _TEST = {
+        'url': 'http://www.soul-anime.net/watching/seirei-tsukai-no-blade-dance-episode-9/',
+        'md5': '05fae04abf72298098b528e98abf4298',
+        'info_dict': {
+            'id': 'seirei-tsukai-no-blade-dance-episode-9',
+            'ext': 'mp4',
+            'title': 'seirei-tsukai-no-blade-dance-episode-9',
+            'description': 'seirei-tsukai-no-blade-dance-episode-9'
+        }
+    }
+    _VALID_URL = r'http://[w.]*soul-anime\.(?P<domain>[^/]+)/watch[^/]*/(?P<id>[^/]+)'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        domain = mobj.group('domain')
+
+        page = self._download_webpage(url, video_id)
+
+        video_url_encoded = self._html_search_regex(
+            r'<div id="download">[^<]*<a href="(?P<url>[^"]+)"', page, 'url')
+        video_url = "http://www.soul-anime." + domain + video_url_encoded
+
+        vid = self._request_webpage(video_url, video_id)
+        ext = vid.info().gettype().split("/")[1]
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'ext': ext,
+            'title': video_id,
+            'description': video_id
+        }
+
+
+class SoulAnimeSeriesIE(InfoExtractor):
+    IE_NAME = "soulanime:series"
+    IE_DESC = "SoulAnime Series"
+
+    _VALID_URL = r'http://[w.]*soul-anime\.(?P<domain>[^/]+)/anime./(?P<id>[^/]+)'
+
+    _EPISODE_REGEX = r'<option value="(/watch[^/]*/[^"]+)">[^<]*</option>'
+
+    _TEST = {
+        'url': 'http://www.soul-anime.net/anime1/black-rock-shooter-tv/',
+        'info_dict': {
+            'id': 'black-rock-shooter-tv'
+        },
+        'playlist_count': 8
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        series_id = mobj.group('id')
+        domain = mobj.group('domain')
+
+        pattern = re.compile(self._EPISODE_REGEX)
+
+        page = self._download_webpage(url, series_id, "Downloading series page")
+        mobj = pattern.findall(page)
+
+        entries = [self.url_result("http://www.soul-anime." + domain + obj) for obj in mobj]
+
+        return self.playlist_result(entries, series_id)
diff --git a/youtube_dl/extractor/videofun.py b/youtube_dl/extractor/videofun.py
new file mode 100644
index 000000000..0364b9d32
--- /dev/null
+++ b/youtube_dl/extractor/videofun.py
@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urllib_parse
+)
+
+
+class VideoFunIE(InfoExtractor):
+    _VALID_URL = r'http://[w.]*videofun\.me/embed/(?P<id>[0-9a-f]+)'
+
+    _TEST = {
+        'url': 'http://videofun.me/embed/8267659be070860af600fee7deadbcdb?w=600&h=438',
+        'md5': 'e37e99d665f503dd2db952f7c4dba9e6',
+        'info_dict': {
+            'id': 'Mahou-Shoujo-Madoka-Magica-07',
+            'ext': 'flv',
+            'title': 'Mahou-Shoujo-Madoka-Magica-07',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, video_id, 'Downloading video page')
+
+        video_url_encoded = self._html_search_regex(
+            r'url: "(http://gateway\.videofun\.me[^"]+)"', webpage, 'video url')
+        video_url = compat_urllib_parse.unquote(video_url_encoded)
+        title = self._html_search_regex(r'.*/([^.]*)\.', video_url, 'title')
+
+        return {
+            'id': title,
+            'url': video_url,
+            'title': title,
+        }

From 95ceeec72200ed3b2c94a54650eb69dfe946e595 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 02:05:35 +0100
Subject: [PATCH 383/974] Remove unused import

---
 youtube_dl/downloader/mplayer.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py
index 34b23b5c2..72cef30ea 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/mplayer.py
@@ -4,7 +4,6 @@ import os
 import subprocess
 
 from .common import FileDownloader
-from ..compat import compat_subprocess_get_DEVNULL
 from ..utils import (
     check_executable,
     encodeFilename,

From c11125f9ed952f9b7ebd06c15eacadcc6005dd8c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 02:06:53 +0100
Subject: [PATCH 384/974] [tests] Remove format 138 from tests (#4559)

---
 test/test_YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index f8e4f930e..730f7ec26 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -218,7 +218,7 @@ class TestFormatSelection(unittest.TestCase):
             # 3D
             '85', '84', '102', '83', '101', '82', '100',
             # Dash video
-            '138', '137', '248', '136', '247', '135', '246',
+            '137', '248', '136', '247', '135', '246',
             '245', '244', '134', '243', '133', '242', '160',
             # Dash audio
             '141', '172', '140', '171', '139',

From 8848314c08284f6a4b8f3c3529bf2e3f1b72610c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 02:08:18 +0100
Subject: [PATCH 385/974] [Makefile] Make offline tests actually work offline

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 71470eedb..e53a367ef 100644
--- a/Makefile
+++ b/Makefile
@@ -46,7 +46,7 @@ test:
 ot: offlinetest
 
 offlinetest: codetest
-	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles --exclude test_write_annotations
+	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles --exclude test_write_annotations --exclude test_youtube_lists
 
 tar: youtube-dl.tar.gz
 

From 2ccd1b10e58cc8e5173dc1aeedc2b3f0ef9b55bf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 02:20:45 +0100
Subject: [PATCH 386/974] [soulanime] Fix under Python 3

---
 youtube_dl/extractor/soulanime.py | 10 ++++++++--
 youtube_dl/utils.py               | 11 +++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soulanime.py b/youtube_dl/extractor/soulanime.py
index 7adb10c03..feef33e27 100644
--- a/youtube_dl/extractor/soulanime.py
+++ b/youtube_dl/extractor/soulanime.py
@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    HEADRequest,
+    urlhandle_detect_ext,
+)
 
 
 class SoulAnimeWatchingIE(InfoExtractor):
@@ -31,8 +35,10 @@ class SoulAnimeWatchingIE(InfoExtractor):
             r'<div id="download">[^<]*<a href="(?P<url>[^"]+)"', page, 'url')
         video_url = "http://www.soul-anime." + domain + video_url_encoded
 
-        vid = self._request_webpage(video_url, video_id)
-        ext = vid.info().gettype().split("/")[1]
+        ext_req = HEADRequest(video_url)
+        ext_handle = self._request_webpage(
+            ext_req, video_id, note='Determining extension')
+        ext = urlhandle_detect_ext(ext_handle)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index efbe64fb3..bdfe053a7 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1550,3 +1550,14 @@ def ytdl_is_updateable():
 def args_to_str(args):
     # Get a short string representation for a subprocess command
     return ' '.join(shlex_quote(a) for a in args)
+
+
+def urlhandle_detect_ext(url_handle):
+    try:
+        url_handle.headers
+        getheader = lambda h: url_handle.headers[h]
+    except AttributeError:  # Python < 3
+        getheader = url_handle.info().getheader
+
+    return getheader('Content-Type').split("/")[1]
+

From 7a1818c99b1729796f62c341b1b3f809cd47dbf8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 03:15:27 +0100
Subject: [PATCH 387/974] [vk] Add support for rutube embeds (Fixes #4514)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rutube.py   | 31 +++++++++++++++++++++++++++++++
 youtube_dl/extractor/vk.py       |  9 +++++++++
 3 files changed, 41 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c3dc09f75..143cd5c49 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -359,6 +359,7 @@ from .ruhd import RUHDIE
 from .rutube import (
     RutubeIE,
     RutubeChannelIE,
+    RutubeEmbedIE,
     RutubeMovieIE,
     RutubePersonIE,
 )
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index b72b5a586..5b1c3577a 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -70,6 +70,37 @@ class RutubeIE(InfoExtractor):
         }
 
 
+class RutubeEmbedIE(InfoExtractor):
+    IE_NAME = 'rutube:embed'
+    IE_DESC = 'Rutube embedded videos'
+    _VALID_URL = 'https?://rutube\.ru/video/embed/(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=',
+        'info_dict': {
+            'id': 'a10e53b86e8f349080f718582ce4c661',
+            'ext': 'mp4',
+            'upload_date': '20131223',
+            'uploader_id': '297833',
+            'description': 'Видео группы ★http://vk.com/foxkidsreset★ музей Fox Kids и Jetix<br/><br/> восстановлено и сделано в шикоформате subziro89 http://vk.com/subziro89',
+            'uploader': 'subziro89 ILya',
+            'title': 'Мистический городок Эйри в Индиан 5 серия озвучка subziro89',
+        },
+        'params': {
+            'skip_download': 'Requires ffmpeg',
+        },
+    }
+
+    def _real_extract(self, url):
+        embed_id = self._match_id(url)
+        webpage = self._download_webpage(url, embed_id)
+
+        canonical_url = self._html_search_regex(
+            r'<link\s+rel="canonical"\s+href="([^"]+?)"', webpage,
+            'Canonical URL')
+        return self.url_result(canonical_url, 'Rutube')
+
+
 class RutubeChannelIE(InfoExtractor):
     IE_NAME = 'rutube:channel'
     IE_DESC = 'Rutube channels'
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 542e9198a..129de6cf3 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -164,6 +164,15 @@ class VKIE(InfoExtractor):
             self.to_screen('Youtube video detected')
             return self.url_result(m_yt.group(1), 'Youtube')
 
+        m_rutube = re.search(
+            r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
+        assert m_rutube
+        if m_rutube is not None:
+            self.to_screen('rutube video detected')
+            rutube_url = self._proto_relative_url(
+                m_rutube.group(1).replace('\\', ''))
+            return self.url_result(rutube_url)
+
         m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.*?});', info_page)
         if m_opts:
             m_opts_url = re.search(r"url\s*:\s*'([^']+)", m_opts.group(1))

From 26886e6140a684058064c30237ef096332e1510f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 03:15:48 +0100
Subject: [PATCH 388/974] release 2015.01.04

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2a54b9bbe..09813928a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.03'
+__version__ = '2015.01.04'

From f4858a71035549cf82b258d01dda5060aef707b7 Mon Sep 17 00:00:00 2001
From: Christopher Krooss <c.krooss@gmail.com>
Date: Sun, 4 Jan 2015 13:33:26 +0100
Subject: [PATCH 389/974] Add support for Radio Bremen

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/radiobremen.py | 55 +++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/radiobremen.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 143cd5c49..349f4fe71 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -339,6 +339,7 @@ from .prosiebensat1 import ProSiebenSat1IE
 from .pyvideo import PyvideoIE
 from .quickvid import QuickVidIE
 from .radiode import RadioDeIE
+from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
new file mode 100644
index 000000000..68c78c4f9
--- /dev/null
+++ b/youtube_dl/extractor/radiobremen.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class RadioBremenIE(InfoExtractor):
+    _VALID_URL = r'http?://(?:www\.)?radiobremen\.de/mediathek/(index\.html)?\?id=(?P<video_id>[0-9]+)'
+    IE_NAME = 'radiobremen'
+
+    _TEST = {
+        'url': 'http://www.radiobremen.de/mediathek/index.html?id=114720',
+        'info_dict': {
+            'id': '114720',
+            'ext': 'mp4',
+            'height': 288,
+            'width': 512,
+            'title': 'buten un binnen vom 22. Dezember',
+            'description': 'Unter anderem mit diesen Themen: 45 Flüchtlinge sind in Worpswede angekommen +++ Freies Internet für alle: Bremer arbeiten an einem flächendeckenden W-Lan-Netzwerk +++ Aktivisten kämpfen für das Unibad +++ So war das Wetter 2014 +++',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('video_id')
+
+        meta_url = "http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s" % video_id
+        meta_doc = self._download_webpage(meta_url, video_id, 'Downloading metadata')
+        title = self._html_search_regex("<h1.*>(?P<title>.+)</h1>", meta_doc, "title")
+        description = self._html_search_regex("<p>(?P<description>.*)</p>", meta_doc, "description")
+        duration = self._html_search_regex("L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>", meta_doc, "duration")
+
+        page_doc = self._download_webpage(url, video_id, 'Downloading video information')
+        pattern = "ardformatplayerclassic\(\'playerbereich\',\'(?P<width>[0-9]+)\',\'.*\',\'(?P<video_id>[0-9]+)\',\'(?P<secret>[0-9]+)\',\'(?P<thumbnail>.+)\',\'\'\)"
+        mobj = re.search(pattern, page_doc)
+        width, video_id, secret, thumbnail = int(mobj.group("width")), mobj.group("video_id"), mobj.group("secret"), mobj.group("thumbnail")
+        video_url = "http://dl-ondemand.radiobremen.de/mediabase/{:}/{:}_{:}_{:}.mp4".format(video_id, video_id, secret, width)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'formats': [
+                {'url': video_url,
+                 'ext': 'mp4',
+                 'width': width,
+                 'protocol': 'http'
+                 }
+            ],
+            'thumbnail': thumbnail,
+        }

From 63948fc62c7f0bfcfe7b2ce102ab6e4e87de558c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 4 Jan 2015 13:40:30 +0100
Subject: [PATCH 390/974] [downloader/hls] Respect the 'prefer_ffmpeg' option

---
 youtube_dl/downloader/hls.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 5bb0f3cfd..aa58b52ab 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -11,7 +11,6 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
-    check_executable,
     encodeFilename,
 )
 
@@ -27,16 +26,13 @@ class HlsFD(FileDownloader):
             '-bsf:a', 'aac_adtstoasc',
             encodeFilename(tmpfilename, for_subprocess=True)]
 
-        for program in ['avconv', 'ffmpeg']:
-            if check_executable(program, ['-version']):
-                break
-        else:
+        ffpp = FFmpegPostProcessor(downloader=self)
+        program = ffpp._executable
+        if program is None:
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
-        cmd = [program] + args
-
-        ffpp = FFmpegPostProcessor(downloader=self)
         ffpp.check_version()
+        cmd = [program] + args
 
         retval = subprocess.call(cmd)
         if retval == 0:

From bc1fc5ddbcba784778cbdd98c051ff2493178515 Mon Sep 17 00:00:00 2001
From: Christopher Krooss <c.krooss@gmail.com>
Date: Sun, 4 Jan 2015 14:02:07 +0100
Subject: [PATCH 391/974] Don't check for height as it's not provided

---
 youtube_dl/extractor/radiobremen.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
index 68c78c4f9..6d130d3d9 100644
--- a/youtube_dl/extractor/radiobremen.py
+++ b/youtube_dl/extractor/radiobremen.py
@@ -16,7 +16,6 @@ class RadioBremenIE(InfoExtractor):
         'info_dict': {
             'id': '114720',
             'ext': 'mp4',
-            'height': 288,
             'width': 512,
             'title': 'buten un binnen vom 22. Dezember',
             'description': 'Unter anderem mit diesen Themen: 45 Flüchtlinge sind in Worpswede angekommen +++ Freies Internet für alle: Bremer arbeiten an einem flächendeckenden W-Lan-Netzwerk +++ Aktivisten kämpfen für das Unibad +++ So war das Wetter 2014 +++',

From bc3e582fe457f9239dc3a3386cbfd0e7db167404 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 4 Jan 2015 14:02:17 +0100
Subject: [PATCH 392/974] Don't use '-shortest' option for merging formats
 (closes #4220, closes #4580)

With avconv and older versions of ffmpeg the video is partially copied.
The duration difference between the audio and the video seem to be really small, so it's probably not noticeable.
---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 048525efc..473536dcc 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -520,7 +520,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 class FFmpegMergerPP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
-        args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0', '-shortest']
+        args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0']
         self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename)
         self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
         return True, info

From 9fda6ee39fa2da1949af5e9b95633e3df3c6f6b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 4 Jan 2015 14:06:23 +0100
Subject: [PATCH 393/974] [tf1] Remove unused import

---
 youtube_dl/extractor/tf1.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 07cc81226..025d0877c 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 

From 1d2d0e3ff2b4e55810039caf267bb9ad086f3610 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 4 Jan 2015 14:07:06 +0100
Subject: [PATCH 394/974] utils: Remove blank line at the end of file

---
 youtube_dl/utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bdfe053a7..d4951c406 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1560,4 +1560,3 @@ def urlhandle_detect_ext(url_handle):
         getheader = url_handle.info().getheader
 
     return getheader('Content-Type').split("/")[1]
-

From 67c2bcdf4cf83f9ac32e5f1f50a8b4b38d2ac624 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 19:19:15 +0100
Subject: [PATCH 395/974] Remove extractors which infringe copyright (#4554)

---
 youtube_dl/extractor/__init__.py  |  19 ----
 youtube_dl/extractor/gogoanime.py |  76 ---------------
 youtube_dl/extractor/play44.py    | 149 ------------------------------
 youtube_dl/extractor/videofun.py  |  36 --------
 4 files changed, 280 deletions(-)
 delete mode 100644 youtube_dl/extractor/gogoanime.py
 delete mode 100644 youtube_dl/extractor/play44.py
 delete mode 100644 youtube_dl/extractor/videofun.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 143cd5c49..613e8e05b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -164,10 +164,6 @@ from .globo import GloboIE
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
-from .gogoanime import (
-    GoGoAnimeIE,
-    GoGoAnimeSearchIE
-)
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
@@ -317,16 +313,6 @@ from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .planetaplay import PlanetaPlayIE
 from .played import PlayedIE
-from .play44 import (
-    Play44IE,
-    ByZooIE,
-    Video44IE,
-    VideoWingIE,
-    PlayPandaIE,
-    VideoZooIE,
-    PlayBBIE,
-    EasyVideoIE
-)
 from .playfm import PlayFMIE
 from .playvid import PlayvidIE
 from .podomatic import PodomaticIE
@@ -388,10 +374,6 @@ from .smotri import (
 from .snotr import SnotrIE
 from .sockshare import SockshareIE
 from .sohu import SohuIE
-from .soulanime import (
-    SoulAnimeWatchingIE,
-    SoulAnimeSeriesIE
-)
 from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
@@ -486,7 +468,6 @@ from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
 from .videolecturesnet import VideoLecturesNetIE
-from .videofun import VideoFunIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
 from .videopremium import VideoPremiumIE
diff --git a/youtube_dl/extractor/gogoanime.py b/youtube_dl/extractor/gogoanime.py
deleted file mode 100644
index d4f4ecc58..000000000
--- a/youtube_dl/extractor/gogoanime.py
+++ /dev/null
@@ -1,76 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    compat_urllib_parse,
-    get_element_by_attribute,
-    unescapeHTML
-)
-
-
-class GoGoAnimeIE(InfoExtractor):
-    IE_NAME = 'gogoanime'
-    IE_DESC = 'GoGoAnime'
-    _VALID_URL = r'http://www.gogoanime.com/(?P<id>[A-Za-z0-9-]+)'
-
-    _TEST = {
-        'url': 'http://www.gogoanime.com/mahou-shoujo-madoka-magica-movie-1',
-        'info_dict': {
-            'id': 'mahou-shoujo-madoka-magica-movie-1'
-        },
-        'playlist_count': 3
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
-
-        if 'Oops! Page Not Found</font>' in page:
-            raise ExtractorError('Video does not exist', expected=True)
-
-        content = get_element_by_attribute("class", "postcontent", page)
-        vids = re.findall(r'<iframe[^>]*?src=[\'"](h[^\'"]+)[\'"]', content)
-        vids = [
-            unescapeHTML(compat_urllib_parse.unquote(x))
-            for x in vids if not re.search(r".*videofun.*", x)]
-
-        if re.search(r'<div class="postcontent">[^<]*<p><iframe src=[\'"][^>]+></iframe><br />', page):
-            return self.playlist_result([self.url_result(vid) for vid in vids], video_id)
-
-        title = self._html_search_regex(
-            r'<div class="postdesc">[^<]*<h1>([^<]+)</h1>', page, 'title')
-
-        return {
-            '_type': 'url',
-            'id': video_id,
-            'url': vids[0],
-            'title': title,
-        }
-
-
-class GoGoAnimeSearchIE(InfoExtractor):
-    IE_NAME = 'gogoanime:search'
-    IE_DESC = 'GoGoAnime Search'
-
-    _VALID_URL = r'http://www\.gogoanime\.com/.*\?s=(?P<id>[^&]*)'
-    _TEST = {
-        'url': 'http://www.gogoanime.com/?s=bokusatsu',
-        'info_dict': {
-            'id': 'bokusatsu'
-        },
-        'playlist_count': 6
-    }
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        webpage = self._download_webpage(url, playlist_id)
-
-        posts = re.findall(
-            r'<div class="postlist">[^<]*<p[^>]*>[^<]*<a href="(?P<url>[^"]+)"',
-            webpage)
-
-        return self.playlist_result(
-            [self.url_result(p) for p in posts], playlist_id)
diff --git a/youtube_dl/extractor/play44.py b/youtube_dl/extractor/play44.py
deleted file mode 100644
index b8696e516..000000000
--- a/youtube_dl/extractor/play44.py
+++ /dev/null
@@ -1,149 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    compat_urllib_parse
-)
-
-
-class Play44IE(InfoExtractor):
-    _VALID_URL = r'http://[w.]*play44\.net/embed\.php[^/]*/(?P<id>.+)'
-
-    _TESTS = [{
-        'url': 'http://play44.net/embed.php?w=600&h=438&vid=M/mahou-shoujo-madoka-magica-07.flv',
-        'md5': 'e37e99d665f503dd2db952f7c4dba9e6',
-        'info_dict': {
-            'id': 'mahou-shoujo-madoka-magica-07',
-            'ext': 'flv',
-            'title': 'mahou-shoujo-madoka-magica-07',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
-
-        video_url = compat_urllib_parse.unquote(self._html_search_regex(
-            r'_url = "(https?://[^"]+?)";', page, 'url'))
-        title = self._search_regex(r'.*/(?P<title>[^.]*).', video_url, 'title')
-
-        return {
-            'id': title,
-            'url': video_url,
-            'title': title,
-        }
-
-
-class ByZooIE(Play44IE):
-    _VALID_URL = r'http://[w.]*byzoo\.org/embed\.php[^/]*/(?P<id>.+)'
-
-    _TESTS = [{
-        'url': 'http://byzoo.org/embed.php?w=600&h=438&vid=at/nw/mahou_shoujo_madoka_magica_movie_3_-_part1.mp4',
-        'md5': '455c83dabe2cd9fd74a87612b01fe017',
-        'info_dict': {
-            'id': 'mahou_shoujo_madoka_magica_movie_3_-_part1',
-            'ext': 'mp4',
-            'title': 'mahou_shoujo_madoka_magica_movie_3_-_part1',
-        }
-    }]
-
-
-class Video44IE(Play44IE):
-    _VALID_URL = r'http://[w.]*video44\.net/.*file=(?P<id>[^&].).*'
-
-    _TESTS = [{
-        'url': 'http://www.video44.net/gogo/?w=600&h=438&file=chaoshead-12.flv&sv=1',
-        'md5': '43eaec6d0beb10e8d42459b9f108aff3',
-        'info_dict': {
-            'id': 'chaoshead-12',
-            'ext': 'mp4',
-            'title': 'chaoshead-12',
-        }
-    }]
-
-
-class VideoWingIE(Play44IE):
-    _VALID_URL = r'''(?x)
-        http://[w.]*videowing\.[^/]*/
-        (?:
-            .*video=/*
-            |embed/
-        )
-        (?P<id>[^&?.]+)
-    '''
-
-    _TESTS = [{
-        'url': 'http://videowing.me/embed?w=718&h=438&video=ongoing/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
-        'md5': '4ed320e353ed26c742c4f12a9c210b60',
-        'info_dict': {
-            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
-            'ext': 'mp4',
-            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
-        }
-    }, {
-        'url': 'http://videowing.me/embed/a8d6a39522df066bd734a69f2334497e?w=600&h=438',
-        'md5': '33fdd71581357018c226f95c5cedcfd7',
-        'info_dict': {
-            'id': 'mahoushoujomadokamagicamovie1part1',
-            'ext': 'flv',
-            'title': 'mahoushoujomadokamagicamovie1part1',
-        }
-    }]
-
-
-class PlayPandaIE(Play44IE):
-    _VALID_URL = r'http://[w.]*playpanda\.[^/]*/.*vid=/*(?P<id>[^&].).*'
-
-    _TESTS = [{
-        'url': 'http://playpanda.net/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
-        'md5': '4ed320e353ed26c742c4f12a9c210b60',
-        'info_dict': {
-            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
-            'ext': 'mp4',
-            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
-            'description': 'boku_wa_tomodachi_ga_sukunai_-_05'
-        }
-    }]
-
-
-class VideoZooIE(Play44IE):
-    _VALID_URL = r'http://[w.]*videozoo\.[^/]*/.*vid=/*(?P<id>[^&].).*'
-
-    _TESTS = [{
-        'url': 'http://videozoo.me/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
-        'md5': '4ed320e353ed26c742c4f12a9c210b60',
-        'info_dict': {
-            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
-            'ext': 'mp4',
-            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
-        }
-    }]
-
-
-class PlayBBIE(Play44IE):
-    _VALID_URL = r'http://[w.]*playbb\.[^/]*/.*vid=/*(?P<id>[^&].).*'
-
-    _TESTS = [{
-        'url': 'http://playbb.me/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
-        'md5': '4ed320e353ed26c742c4f12a9c210b60',
-        'info_dict': {
-            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
-            'ext': 'mp4',
-            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
-        }
-    }]
-
-
-class EasyVideoIE(Play44IE):
-    _VALID_URL = r'http://[w.]*easyvideo\.[^/]*/.*file=/*(?P<id>[^&.]+)'
-
-    _TESTS = [{
-        'url': 'http://easyvideo.me/gogo/?w=718&h=438&file=bokuwatomodachigasukunai-04.flv&sv=1',
-        'md5': '26178b57629b7650106d72b191137176',
-        'info_dict': {
-            'id': 'bokuwatomodachigasukunai-04',
-            'ext': 'mp4',
-            'title': 'bokuwatomodachigasukunai-04',
-        },
-        'skip': 'Blocked in Germany',
-    }]
diff --git a/youtube_dl/extractor/videofun.py b/youtube_dl/extractor/videofun.py
deleted file mode 100644
index 0364b9d32..000000000
--- a/youtube_dl/extractor/videofun.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    compat_urllib_parse
-)
-
-
-class VideoFunIE(InfoExtractor):
-    _VALID_URL = r'http://[w.]*videofun\.me/embed/(?P<id>[0-9a-f]+)'
-
-    _TEST = {
-        'url': 'http://videofun.me/embed/8267659be070860af600fee7deadbcdb?w=600&h=438',
-        'md5': 'e37e99d665f503dd2db952f7c4dba9e6',
-        'info_dict': {
-            'id': 'Mahou-Shoujo-Madoka-Magica-07',
-            'ext': 'flv',
-            'title': 'Mahou-Shoujo-Madoka-Magica-07',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(
-            url, video_id, 'Downloading video page')
-
-        video_url_encoded = self._html_search_regex(
-            r'url: "(http://gateway\.videofun\.me[^"]+)"', webpage, 'video url')
-        video_url = compat_urllib_parse.unquote(video_url_encoded)
-        title = self._html_search_regex(r'.*/([^.]*)\.', video_url, 'title')
-
-        return {
-            'id': title,
-            'url': video_url,
-            'title': title,
-        }

From 2f985f4bb4938ee13356bda0436fde18f8c0e434 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 00:18:43 +0100
Subject: [PATCH 396/974] [youtube:toplist] Remove extractor

They use now normal playlists (their id is PL*).
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/youtube.py  | 46 --------------------------------
 2 files changed, 47 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 613e8e05b..79e6bba45 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -546,7 +546,6 @@ from .youtube import (
     YoutubeSearchURLIE,
     YoutubeShowIE,
     YoutubeSubscriptionsIE,
-    YoutubeTopListIE,
     YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeUserIE,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e9bf39a00..d1bbf0b01 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1206,9 +1206,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         if playlist_id.startswith('RD'):
             # Mixes require a custom extraction process
             return self._extract_mix(playlist_id)
-        if playlist_id.startswith('TL'):
-            raise ExtractorError('For downloading YouTube.com top lists, use '
-                                 'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
 
         url = self._TEMPLATE_URL % playlist_id
         page = self._download_webpage(url, playlist_id)
@@ -1254,49 +1251,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         return self.playlist_result(url_results, playlist_id, playlist_title)
 
 
-class YoutubeTopListIE(YoutubePlaylistIE):
-    IE_NAME = 'youtube:toplist'
-    IE_DESC = ('YouTube.com top lists, "yttoplist:{channel}:{list title}"'
-               ' (Example: "yttoplist:music:Top Tracks")')
-    _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$'
-    _TESTS = [{
-        'url': 'yttoplist:music:Trending',
-        'playlist_mincount': 5,
-        'skip': 'Only works for logged-in users',
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        channel = mobj.group('chann')
-        title = mobj.group('title')
-        query = compat_urllib_parse.urlencode({'title': title})
-        channel_page = self._download_webpage(
-            'https://www.youtube.com/%s' % channel, title)
-        link = self._html_search_regex(
-            r'''(?x)
-                <a\s+href="([^"]+)".*?>\s*
-                <span\s+class="branded-page-module-title-text">\s*
-                <span[^>]*>.*?%s.*?</span>''' % re.escape(query),
-            channel_page, 'list')
-        url = compat_urlparse.urljoin('https://www.youtube.com/', link)
-
-        video_re = r'data-index="\d+".*?data-video-id="([0-9A-Za-z_-]{11})"'
-        ids = []
-        # sometimes the webpage doesn't contain the videos
-        # retry until we get them
-        for i in itertools.count(0):
-            msg = 'Downloading Youtube mix'
-            if i > 0:
-                msg += ', retry #%d' % i
-
-            webpage = self._download_webpage(url, title, msg)
-            ids = orderedSet(re.findall(video_re, webpage))
-            if ids:
-                break
-        url_results = self._ids_to_results(ids)
-        return self.playlist_result(url_results, playlist_title=title)
-
-
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = 'YouTube.com channels'
     _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'

From caf90bfaa5434d9ff7035d8575b842b076178ca3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 5 Jan 2015 02:22:01 +0200
Subject: [PATCH 397/974] [webofstories] Add new extractor (Closes #4585)

---
 youtube_dl/extractor/__init__.py     |   1 +
 youtube_dl/extractor/webofstories.py | 102 +++++++++++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 youtube_dl/extractor/webofstories.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79e6bba45..0c8729384 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -511,6 +511,7 @@ from .wdr import (
     WDRMobileIE,
     WDRMausIE,
 )
+from .webofstories import WebOfStoriesIE
 from .weibo import WeiboIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
new file mode 100644
index 000000000..396cf4e83
--- /dev/null
+++ b/youtube_dl/extractor/webofstories.py
@@ -0,0 +1,102 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class WebOfStoriesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?webofstories\.com/play/(?:[^/]+/)?(?P<id>[0-9]+)'
+    _VIDEO_DOMAIN = 'http://eu-mobile.webofstories.com/'
+    _GREAT_LIFE_STREAMER = 'rtmp://eu-cdn1.webofstories.com/cfx/st/'
+    _USER_STREAMER = 'rtmp://eu-users.webofstories.com/cfx/st/'
+    _TESTS = [
+        {
+            'url': 'http://www.webofstories.com/play/hans.bethe/71',
+            'md5': '373e4dd915f60cfe3116322642ddf364',
+            'info_dict': {
+                'id': '4536',
+                'ext': 'mp4',
+                'title': 'The temperature of the sun',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'Hans Bethe talks about calculating the temperature of the sun',
+                'duration': 238,
+            }
+        },
+        {
+            'url': 'http://www.webofstories.com/play/55908',
+            'md5': '2985a698e1fe3211022422c4b5ed962c',
+            'info_dict': {
+                'id': '55908',
+                'ext': 'mp4',
+                'title': 'The story of Gemmata obscuriglobus',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'Planctomycete talks about The story of Gemmata obscuriglobus',
+                'duration': 169,
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._og_search_title(webpage)
+        description = self._html_search_meta('description', webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        story_filename = self._search_regex(
+            r'\.storyFileName\("([^"]+)"\)', webpage, 'story filename')
+        speaker_id = self._search_regex(
+            r'\.speakerId\("([^"]+)"\)', webpage, 'speaker ID')
+        story_id = self._search_regex(
+            r'\.storyId\((\d+)\)', webpage, 'story ID')
+        speaker_type = self._search_regex(
+            r'\.speakerType\("([^"]+)"\)', webpage, 'speaker type')
+        great_life = self._search_regex(
+            r'isGreatLifeStory\s*=\s*(true|false)', webpage, 'great life story')
+        is_great_life_series = great_life == 'true'
+        duration = int_or_none(self._search_regex(
+            r'\.duration\((\d+)\)', webpage, 'duration', fatal=False))
+
+        # URL building, see: http://www.webofstories.com/scripts/player.js
+        ms_prefix = ''
+        if speaker_type.lower() == 'ms':
+            ms_prefix = 'mini_sites/'
+
+        if is_great_life_series:
+            mp4_url = '{0:}lives/{1:}/{2:}.mp4'.format(
+                self._VIDEO_DOMAIN, speaker_id, story_filename)
+            rtmp_ext = 'flv'
+            streamer = self._GREAT_LIFE_STREAMER
+            play_path = 'stories/{0:}/{1:}'.format(
+                speaker_id, story_filename)
+        else:
+            mp4_url = '{0:}{1:}{2:}/{3:}.mp4'.format(
+                self._VIDEO_DOMAIN, ms_prefix, speaker_id, story_filename)
+            rtmp_ext = 'mp4'
+            streamer = self._USER_STREAMER
+            play_path = 'mp4:{0:}{1:}/{2}.mp4'.format(
+                ms_prefix, speaker_id, story_filename)
+
+        formats = [{
+            'format_id': 'mp4_sd',
+            'url': mp4_url,
+        }, {
+            'format_id': 'rtmp_sd',
+            'page_url': url,
+            'url': streamer,
+            'ext': rtmp_ext,
+            'play_path': play_path,
+        }]
+
+        self._sort_formats(formats)
+
+        return {
+            'id': story_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'description': description,
+            'duration': duration,
+        }

From adf3c58ad31e7376f085271a02fdfe56b1e75989 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 5 Jan 2015 02:55:12 +0200
Subject: [PATCH 398/974] [lrt] Fix missing provider key

Also, modernize a bit.
---
 youtube_dl/extractor/lrt.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index d72d470aa..9c2fbdd96 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..utils import (
@@ -28,7 +27,6 @@ class LRTIE(InfoExtractor):
         'params': {
             'skip_download': True,  # HLS download
         },
-
     }
 
     def _real_extract(self, url):
@@ -44,7 +42,9 @@ class LRTIE(InfoExtractor):
 
         formats = []
         for js in re.findall(r'(?s)config:\s*(\{.*?\})', webpage):
-            data = json.loads(js_to_json(js))
+            data = self._parse_json(js, video_id, transform_source=js_to_json)
+            if 'provider' not in data:
+                continue
             if data['provider'] == 'rtmp':
                 formats.append({
                     'format_id': 'rtmp',

From bdf80aa542da15437545ae9c17cd5c80e17e171f Mon Sep 17 00:00:00 2001
From: Bart Kappenburg <bartkappenburg@gmail.com>
Date: Mon, 5 Jan 2015 11:51:24 +0100
Subject: [PATCH 399/974] Update rtlnl.py

Added support for the non-www version of rtlxl.nl by making "www." optional.
---
 youtube_dl/extractor/rtlnl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index d029b0ec5..a3ca79f2c 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -8,7 +8,7 @@ from ..utils import parse_duration
 
 class RtlXlIE(InfoExtractor):
     IE_NAME = 'rtlxl.nl'
-    _VALID_URL = r'https?://www\.rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)'
+    _VALID_URL = r'https?://(www\.)?rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)'
 
     _TEST = {
         'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677',

From a4c3f486394ae8ead64e8e634433670639e3080f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 11:46:40 +0100
Subject: [PATCH 400/974] [vimple] Replace tests

The first one seems to be no longer available and the second was an episode from a tv show.
---
 youtube_dl/extractor/vimple.py | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/vimple.py b/youtube_dl/extractor/vimple.py
index 33d370e1c..ee3d86117 100644
--- a/youtube_dl/extractor/vimple.py
+++ b/youtube_dl/extractor/vimple.py
@@ -14,28 +14,17 @@ class VimpleIE(InfoExtractor):
     IE_DESC = 'Vimple.ru'
     _VALID_URL = r'https?://(player.vimple.ru/iframe|vimple.ru)/(?P<id>[a-f0-9]{10,})'
     _TESTS = [
-        # Quality: Large, from iframe
         {
-            'url': 'http://player.vimple.ru/iframe/b132bdfd71b546d3972f9ab9a25f201c',
+            'url': 'http://vimple.ru/c0f6b1687dcd4000a97ebe70068039cf',
+            'md5': '2e750a330ed211d3fd41821c6ad9a279',
             'info_dict': {
-                'id': 'b132bdfd71b546d3972f9ab9a25f201c',
-                'title': 'great-escape-minecraft.flv',
+                'id': 'c0f6b1687dcd4000a97ebe70068039cf',
                 'ext': 'mp4',
-                'duration': 352,
-                'webpage_url': 'http://vimple.ru/b132bdfd71b546d3972f9ab9a25f201c',
+                'title': 'Sunset',
+                'duration': 20,
+                'thumbnail': 're:https?://.*?\.jpg',
             },
         },
-        # Quality: Medium, from mainpage
-        {
-            'url': 'http://vimple.ru/a15950562888453b8e6f9572dc8600cd',
-            'info_dict': {
-                'id': 'a15950562888453b8e6f9572dc8600cd',
-                'title': 'DB 01',
-                'ext': 'flv',
-                'duration': 1484,
-                'webpage_url': 'http://vimple.ru/a15950562888453b8e6f9572dc8600cd',
-            }
-        },
     ]
 
     def _real_extract(self, url):

From 628bc4d1e73ddef2b67eb6aba7b642c2e0ea894e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 12:28:35 +0100
Subject: [PATCH 401/974] [khanacademy] Update test

---
 youtube_dl/extractor/khanacademy.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/khanacademy.py b/youtube_dl/extractor/khanacademy.py
index 408d00944..08a671fa8 100644
--- a/youtube_dl/extractor/khanacademy.py
+++ b/youtube_dl/extractor/khanacademy.py
@@ -22,8 +22,10 @@ class KhanAcademyIE(InfoExtractor):
             'description': 'The perfect cipher',
             'duration': 176,
             'uploader': 'Brit Cruise',
+            'uploader_id': 'khanacademy',
             'upload_date': '20120411',
-        }
+        },
+        'add_ie': ['Youtube'],
     }, {
         'url': 'https://www.khanacademy.org/math/applied-math/cryptography',
         'info_dict': {

From 75311a7e160912550e3c07642a5635f85f72cb0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 12:29:32 +0100
Subject: [PATCH 402/974] .travis.yml: Remove my email from the list

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index c6cc7a994..f14014414 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,7 +9,6 @@ notifications:
   email:
     - filippo.valsorda@gmail.com
     - phihag@phihag.de
-    - jaime.marquinez.ferrandiz+travis@gmail.com
     - yasoob.khld@gmail.com
 #  irc:
 #    channels:

From 87830900a95f95308dac565f9da12387edea65e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 13:07:24 +0100
Subject: [PATCH 403/974] [generic] Update some tests

---
 youtube_dl/extractor/generic.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 493afb57d..5c41ff517 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -131,12 +131,13 @@ class GenericIE(InfoExtractor):
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
-            'md5': '5644c6ca5d5782c1d0d350dad9bd840c',
+            'md5': '166dd577b433b4d4ebfee10b0824d8ff',
             'info_dict': {
                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
                 'ext': 'mp4',
                 'title': '2cc213299525360.mov',  # that's what we get
             },
+            'add_ie': ['Ooyala'],
         },
         # google redirect
         {
@@ -146,7 +147,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20130224',
                 'uploader_id': 'TheVerge',
-                'description': 'Chris Ziegler takes a look at the Alcatel OneTouch Fire and the ZTE Open; two of the first Firefox OS handsets to be officially announced.',
+                'description': 're:^Chris Ziegler takes a look at the\.*',
                 'uploader': 'The Verge',
                 'title': 'First Firefox OS phones side-by-side',
             },

From cd791a5ea08b77dab37c15efa7e064c07144cb6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 13:11:13 +0100
Subject: [PATCH 404/974] [ted] Add support for embed-ssl.ted.com embedded
 videos

---
 youtube_dl/extractor/generic.py | 2 +-
 youtube_dl/extractor/ted.py     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5c41ff517..2d871f8b4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -926,7 +926,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded TED player
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>http://embed\.ted\.com/.+?)\1', webpage)
+                r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'TED')
 
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 944177426..10b3b706a 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -13,7 +13,7 @@ from ..compat import (
 class TEDIE(SubtitlesInfoExtractor):
     _VALID_URL = r'''(?x)
         (?P<proto>https?://)
-        (?P<type>www|embed)(?P<urlmain>\.ted\.com/
+        (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
         (
             (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
             |
@@ -98,7 +98,7 @@ class TEDIE(SubtitlesInfoExtractor):
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url, re.VERBOSE)
-        if m.group('type') == 'embed':
+        if m.group('type').startswith('embed'):
             desktop_url = m.group('proto') + 'www' + m.group('urlmain')
             return self.url_result(desktop_url, 'TED')
         name = m.group('name')

From a285b6377b46518ca45d6a41481bf920b353a857 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 13:59:49 +0100
Subject: [PATCH 405/974] [normalboots] Skip download in test, it uses rtmp

---
 youtube_dl/extractor/normalboots.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index 3d35b11ac..c13ff0d65 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -22,7 +22,11 @@ class NormalbootsIE(InfoExtractor):
             'description': 'Jon is late for Christmas. Typical. Thanks to: Paul Ritchey for Co-Writing/Filming: http://www.youtube.com/user/ContinueShow Michael Azzi for Christmas Intro Animation: http://michafrar.tumblr.com/ Jerrod Waters for Christmas Intro Music: http://www.youtube.com/user/xXJerryTerryXx Casey Ormond for ‘Tense Battle Theme’:\xa0http://www.youtube.com/Kiamet/',
             'uploader': 'JonTron',
             'upload_date': '20140125',
-        }
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From 03ff2cc1c49c82daf2218b76e169c2d679447f03 Mon Sep 17 00:00:00 2001
From: oteng <otengkwaku@gmail.com>
Date: Mon, 5 Jan 2015 16:28:24 +0000
Subject: [PATCH 406/974] [Auengine] corrected extractions logic

The way the video download url was been extracted was
not working well so i change it for it to extract the
correct url
---
 .gitignore                       |  2 ++
 youtube_dl/extractor/auengine.py | 16 ++++++----------
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/.gitignore b/.gitignore
index 86312d4e4..0422adf44 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,5 @@ updates_key.pem
 test/testdata
 .tox
 youtube-dl.zsh
+.idea
+.idea/*
\ No newline at end of file
diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
index 014a21952..17c3ad2ef 100644
--- a/youtube_dl/extractor/auengine.py
+++ b/youtube_dl/extractor/auengine.py
@@ -29,17 +29,12 @@ class AUEngineIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         title = self._html_search_regex(r'<title>(?P<title>.+?)</title>', webpage, 'title')
         title = title.strip()
-        links = re.findall(r'\s(?:file|url):\s*["\']([^\'"]+)["\']', webpage)
-        links = map(compat_urllib_parse.unquote, links)
+        video_url = re.findall(r'http://\w+.auengine.com/vod/.*[^\W]', webpage)
+        video_url = map(compat_urllib_parse.unquote, video_url)[0]
+        thumbnail = re.findall(r'http://\w+.auengine.com/thumb/.*[^\W]', webpage)
+        thumbnail = map(compat_urllib_parse.unquote, thumbnail)[0]
 
-        thumbnail = None
-        video_url = None
-        for link in links:
-            if link.endswith('.png'):
-                thumbnail = link
-            elif '/videos/' in link:
-                video_url = link
-        if not video_url:
+        if video_url == "" and thumbnail =="":
             raise ExtractorError('Could not find video URL')
         ext = '.' + determine_ext(video_url)
         if ext == title[-len(ext):]:
@@ -52,3 +47,4 @@ class AUEngineIE(InfoExtractor):
             'thumbnail': thumbnail,
             'http_referer': 'http://www.auengine.com/flowplayer/flowplayer.commercial-3.2.14.swf',
         }
+

From 9d247bbd2d972953fbb9e8f9aee67472d3854883 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:13:19 +0100
Subject: [PATCH 407/974] [radiobremen] Fix under Python 2.6 and fix duration

---
 youtube_dl/extractor/radiobremen.py | 30 +++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
index 6d130d3d9..9f7e6af15 100644
--- a/youtube_dl/extractor/radiobremen.py
+++ b/youtube_dl/extractor/radiobremen.py
@@ -5,10 +5,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import parse_duration
 
 
 class RadioBremenIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?radiobremen\.de/mediathek/(index\.html)?\?id=(?P<video_id>[0-9]+)'
+    _VALID_URL = r'http?://(?:www\.)?radiobremen\.de/mediathek/(?:index\.html)?\?id=(?P<id>[0-9]+)'
     IE_NAME = 'radiobremen'
 
     _TEST = {
@@ -16,6 +17,7 @@ class RadioBremenIE(InfoExtractor):
         'info_dict': {
             'id': '114720',
             'ext': 'mp4',
+            'duration': 1685,
             'width': 512,
             'title': 'buten un binnen vom 22. Dezember',
             'description': 'Unter anderem mit diesen Themen: 45 Flüchtlinge sind in Worpswede angekommen +++ Freies Internet für alle: Bremer arbeiten an einem flächendeckenden W-Lan-Netzwerk +++ Aktivisten kämpfen für das Unibad +++ So war das Wetter 2014 +++',
@@ -23,32 +25,32 @@ class RadioBremenIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
 
         meta_url = "http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s" % video_id
         meta_doc = self._download_webpage(meta_url, video_id, 'Downloading metadata')
         title = self._html_search_regex("<h1.*>(?P<title>.+)</h1>", meta_doc, "title")
         description = self._html_search_regex("<p>(?P<description>.*)</p>", meta_doc, "description")
-        duration = self._html_search_regex("L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>", meta_doc, "duration")
+        duration = parse_duration(
+            self._html_search_regex("L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>", meta_doc, "duration"))
 
         page_doc = self._download_webpage(url, video_id, 'Downloading video information')
         pattern = "ardformatplayerclassic\(\'playerbereich\',\'(?P<width>[0-9]+)\',\'.*\',\'(?P<video_id>[0-9]+)\',\'(?P<secret>[0-9]+)\',\'(?P<thumbnail>.+)\',\'\'\)"
         mobj = re.search(pattern, page_doc)
-        width, video_id, secret, thumbnail = int(mobj.group("width")), mobj.group("video_id"), mobj.group("secret"), mobj.group("thumbnail")
-        video_url = "http://dl-ondemand.radiobremen.de/mediabase/{:}/{:}_{:}_{:}.mp4".format(video_id, video_id, secret, width)
+        video_url = (
+            "http://dl-ondemand.radiobremen.de/mediabase/%s/%s_%s_%s.mp4" %
+            (video_id, video_id, mobj.group("secret"), mobj.group('width')))
 
+        formats = [{
+            'url': video_url,
+            'ext': 'mp4',
+            'width': int(mobj.group("width")),
+        }]
         return {
             'id': video_id,
             'title': title,
             'description': description,
             'duration': duration,
-            'formats': [
-                {'url': video_url,
-                 'ext': 'mp4',
-                 'width': width,
-                 'protocol': 'http'
-                 }
-            ],
-            'thumbnail': thumbnail,
+            'formats': formats,
+            'thumbnail': mobj.group('thumbnail'),
         }

From aa80652f47b3df14664556913d4f14172c9ec4fb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:14:09 +0100
Subject: [PATCH 408/974] [radiobremen] Add test for thumbnail

---
 youtube_dl/extractor/radiobremen.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
index 9f7e6af15..057dc15ab 100644
--- a/youtube_dl/extractor/radiobremen.py
+++ b/youtube_dl/extractor/radiobremen.py
@@ -20,6 +20,7 @@ class RadioBremenIE(InfoExtractor):
             'duration': 1685,
             'width': 512,
             'title': 'buten un binnen vom 22. Dezember',
+            'thumbnail': 're:https?://.*\.jpg$',
             'description': 'Unter anderem mit diesen Themen: 45 Flüchtlinge sind in Worpswede angekommen +++ Freies Internet für alle: Bremer arbeiten an einem flächendeckenden W-Lan-Netzwerk +++ Aktivisten kämpfen für das Unibad +++ So war das Wetter 2014 +++',
         },
     }

From 5e3e1c82d828bc54f6873d2c7bdab315713e9a02 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:14:39 +0100
Subject: [PATCH 409/974] Credit @ckrooss for radiobremen (#4632)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 9b548cf25..a63c97ae0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -99,3 +99,4 @@ Max Reimann
 Cédric Luthi
 Thijs Vermeir
 Joel Leclerc
+Christopher Krooss

From d7cc31b63e1efaf5762f38897d4c717901e127e3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:16:47 +0100
Subject: [PATCH 410/974] [generic] PEP8

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2d871f8b4..7a5bf9392 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -926,7 +926,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded TED player
         mobj = re.search(
-                r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'TED')
 

From dda620e88c68e995afcc3cd35b9d360cb42527a0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:17:03 +0100
Subject: [PATCH 411/974] [radiobremen] Make code more readable and more
 resilient to failures

---
 youtube_dl/extractor/radiobremen.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
index 057dc15ab..0d706312e 100644
--- a/youtube_dl/extractor/radiobremen.py
+++ b/youtube_dl/extractor/radiobremen.py
@@ -29,15 +29,21 @@ class RadioBremenIE(InfoExtractor):
         video_id = self._match_id(url)
 
         meta_url = "http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s" % video_id
-        meta_doc = self._download_webpage(meta_url, video_id, 'Downloading metadata')
-        title = self._html_search_regex("<h1.*>(?P<title>.+)</h1>", meta_doc, "title")
-        description = self._html_search_regex("<p>(?P<description>.*)</p>", meta_doc, "description")
-        duration = parse_duration(
-            self._html_search_regex("L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>", meta_doc, "duration"))
+        meta_doc = self._download_webpage(
+            meta_url, video_id, 'Downloading metadata')
+        title = self._html_search_regex(
+            r"<h1.*>(?P<title>.+)</h1>", meta_doc, "title")
+        description = self._html_search_regex(
+            r"<p>(?P<description>.*)</p>", meta_doc, "description", fatal=False)
+        duration = parse_duration(self._html_search_regex(
+            r"L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>",
+            meta_doc, "duration", fatal=False))
 
-        page_doc = self._download_webpage(url, video_id, 'Downloading video information')
-        pattern = "ardformatplayerclassic\(\'playerbereich\',\'(?P<width>[0-9]+)\',\'.*\',\'(?P<video_id>[0-9]+)\',\'(?P<secret>[0-9]+)\',\'(?P<thumbnail>.+)\',\'\'\)"
-        mobj = re.search(pattern, page_doc)
+        page_doc = self._download_webpage(
+            url, video_id, 'Downloading video information')
+        mobj = re.search(
+            r"ardformatplayerclassic\(\'playerbereich\',\'(?P<width>[0-9]+)\',\'.*\',\'(?P<video_id>[0-9]+)\',\'(?P<secret>[0-9]+)\',\'(?P<thumbnail>.+)\',\'\'\)",
+            page_doc)
         video_url = (
             "http://dl-ondemand.radiobremen.de/mediabase/%s/%s_%s_%s.mp4" %
             (video_id, video_id, mobj.group("secret"), mobj.group('width')))

From 6291438073e35adc94f573a43625fb54a64cf733 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:21:32 +0100
Subject: [PATCH 412/974] [auengine] Simplify (#4643)

---
 youtube_dl/extractor/auengine.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
index 17c3ad2ef..a1b666be0 100644
--- a/youtube_dl/extractor/auengine.py
+++ b/youtube_dl/extractor/auengine.py
@@ -7,6 +7,7 @@ from ..compat import compat_urllib_parse
 from ..utils import (
     determine_ext,
     ExtractorError,
+    remove_end,
 )
 
 
@@ -27,18 +28,18 @@ class AUEngineIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<title>(?P<title>.+?)</title>', webpage, 'title')
-        title = title.strip()
-        video_url = re.findall(r'http://\w+.auengine.com/vod/.*[^\W]', webpage)
-        video_url = map(compat_urllib_parse.unquote, video_url)[0]
-        thumbnail = re.findall(r'http://\w+.auengine.com/thumb/.*[^\W]', webpage)
-        thumbnail = map(compat_urllib_parse.unquote, thumbnail)[0]
+        title = self._html_search_regex(
+            r'<title>\s*(?P<title>.+?)\s*</title>', webpage, 'title')
+        video_urls = re.findall(r'http://\w+.auengine.com/vod/.*[^\W]', webpage)
+        video_url = compat_urllib_parse.unquote(video_urls[0])
+        thumbnails = re.findall(r'http://\w+.auengine.com/thumb/.*[^\W]', webpage)
+        thumbnail = compat_urllib_parse.unquote(thumbnails[0])
 
-        if video_url == "" and thumbnail =="":
+        if not video_url:
             raise ExtractorError('Could not find video URL')
+
         ext = '.' + determine_ext(video_url)
-        if ext == title[-len(ext):]:
-            title = title[:-len(ext)]
+        title = remove_end(title, ext)
 
         return {
             'id': video_id,
@@ -47,4 +48,3 @@ class AUEngineIE(InfoExtractor):
             'thumbnail': thumbnail,
             'http_referer': 'http://www.auengine.com/flowplayer/flowplayer.commercial-3.2.14.swf',
         }
-

From f4bca0b348fe1f4f65c939b496973062180e0c4f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:44:29 +0100
Subject: [PATCH 413/974] release 2015.01.05

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 09813928a..086f0ebf0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.04'
+__version__ = '2015.01.05'

From 8f9529cd0559bdbe6c568cfd765f9129666a77be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 19:14:50 +0100
Subject: [PATCH 414/974] [motorsport] Fix extraction and make trailing '/'
 optional

They directly embed a youtube video now.
---
 youtube_dl/extractor/motorsport.py | 60 ++++++++++++------------------
 1 file changed, 23 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/motorsport.py b/youtube_dl/extractor/motorsport.py
index f5ca74e97..c1a482dba 100644
--- a/youtube_dl/extractor/motorsport.py
+++ b/youtube_dl/extractor/motorsport.py
@@ -1,63 +1,49 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import hashlib
-import json
-import time
-
 from .common import InfoExtractor
 from ..compat import (
-    compat_parse_qs,
-    compat_str,
-)
-from ..utils import (
-    int_or_none,
+    compat_urlparse,
 )
 
 
 class MotorsportIE(InfoExtractor):
     IE_DESC = 'motorsport.com'
-    _VALID_URL = r'http://www\.motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P<id>[^/]+)/(?:$|[?#])'
+    _VALID_URL = r'http://www\.motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P<id>[^/]+)/?(?:$|[?#])'
     _TEST = {
         'url': 'http://www.motorsport.com/f1/video/main-gallery/red-bull-racing-2014-rules-explained/',
-        'md5': '5592cb7c5005d9b2c163df5ac3dc04e4',
         'info_dict': {
-            'id': '7063',
+            'id': '2-T3WuR-KMM',
             'ext': 'mp4',
             'title': 'Red Bull Racing: 2014 Rules Explained',
-            'duration': 207,
+            'duration': 208,
             'description': 'A new clip from Red Bull sees Daniel Ricciardo and Sebastian Vettel explain the 2014 Formula One regulations – which are arguably the most complex the sport has ever seen.',
-            'uploader': 'rainiere',
-            'thumbnail': r're:^http://.*motorsport\.com/.+\.jpg$'
-        }
+            'uploader': 'mcomstaff',
+            'uploader_id': 'UC334JIYKkVnyFoNCclfZtHQ',
+            'upload_date': '20140903',
+            'thumbnail': r're:^https?://.+\.jpg$'
+        },
+        'add_ie': ['Youtube'],
+        'params': {
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        flashvars_code = self._html_search_regex(
-            r'<embed id="player".*?flashvars="([^"]+)"', webpage, 'flashvars')
-        flashvars = compat_parse_qs(flashvars_code)
-        params = json.loads(flashvars['parameters'][0])
-
-        e = compat_str(int(time.time()) + 24 * 60 * 60)
-        base_video_url = params['location'] + '?e=' + e
-        s = 'h3hg713fh32'
-        h = hashlib.md5((s + base_video_url).encode('utf-8')).hexdigest()
-        video_url = base_video_url + '&h=' + h
-
-        uploader = self._html_search_regex(
-            r'(?s)<span class="label">Video by: </span>(.*?)</a>', webpage,
-            'uploader', fatal=False)
+        iframe_path = self._html_search_regex(
+            r'<iframe id="player_iframe"[^>]+src="([^"]+)"', webpage,
+            'iframe path')
+        iframe = self._download_webpage(
+            compat_urlparse.urljoin(url, iframe_path), display_id,
+            'Downloading iframe')
+        youtube_id = self._search_regex(
+            r'www.youtube.com/embed/(.{11})', iframe, 'youtube id')
 
         return {
-            'id': params['video_id'],
+            '_type': 'url_transparent',
             'display_id': display_id,
-            'title': params['title'],
-            'url': video_url,
-            'description': params.get('description'),
-            'thumbnail': params.get('main_thumb'),
-            'duration': int_or_none(params.get('duration')),
-            'uploader': uploader,
+            'url': 'https://youtube.com/watch?v=%s' % youtube_id,
         }

From 3b8f3a1504b46ff6085617ea22e027668d45a503 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Mon, 5 Jan 2015 20:12:29 +0200
Subject: [PATCH 415/974] [downloader/f4m] <metadata> is optional according to
 the F4M specs

---
 youtube_dl/downloader/f4m.py | 38 ++++++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index f9f6f3e73..87d3150c7 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -187,24 +187,27 @@ def build_fragments_list(boot_info):
     return res
 
 
-def write_flv_header(stream, metadata):
-    """Writes the FLV header and the metadata to stream"""
+def write_flv_header(stream):
+    """Writes the FLV header to stream"""
     # FLV header
     stream.write(b'FLV\x01')
     stream.write(b'\x05')
     stream.write(b'\x00\x00\x00\x09')
-    # FLV File body
     stream.write(b'\x00\x00\x00\x00')
-    # FLVTAG
-    # Script data
-    stream.write(b'\x12')
-    # Size of the metadata with 3 bytes
-    stream.write(struct_pack('!L', len(metadata))[1:])
-    stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
-    stream.write(metadata)
-    # Magic numbers extracted from the output files produced by AdobeHDS.php
-    # (https://github.com/K-S-V/Scripts)
-    stream.write(b'\x00\x00\x01\x73')
+
+
+def write_metadata_tag(stream, metadata):
+    """Writes optional metadata tag to stream"""
+    if metadata:
+        # Script data
+        stream.write(b'\x12')
+        # Size of the metadata with 3 bytes
+        stream.write(struct_pack('!L', len(metadata))[1:])
+        stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
+        stream.write(metadata)
+        # Magic numbers extracted from the output files produced by AdobeHDS.php
+        # (https://github.com/K-S-V/Scripts)
+        stream.write(b'\x00\x00\x01\x73')
 
 
 def _add_ns(prop):
@@ -256,7 +259,11 @@ class F4mFD(FileDownloader):
             bootstrap = self.ydl.urlopen(bootstrap_url).read()
         else:
             bootstrap = base64.b64decode(bootstrap_node.text)
-        metadata = base64.b64decode(media.find(_add_ns('metadata')).text)
+        metadata_node = media.find(_add_ns('metadata'))
+        if metadata_node is not None:
+            metadata = base64.b64decode(metadata_node.text)
+        else:
+            metadata = None
         boot_info = read_bootstrap_info(bootstrap)
 
         fragments_list = build_fragments_list(boot_info)
@@ -269,7 +276,8 @@ class F4mFD(FileDownloader):
 
         tmpfilename = self.temp_name(filename)
         (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
-        write_flv_header(dest_stream, metadata)
+        write_flv_header(dest_stream)
+        write_metadata_tag(dest_stream, metadata)
 
         # This dict stores the download progress, it's updated by the progress
         # hook

From 2c322cc5d65550de10d70d812ac3cd6742452252 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Mon, 5 Jan 2015 20:22:17 +0200
Subject: [PATCH 416/974] [downloader/f4m] The last value in a tag is the tag
 length

---
 youtube_dl/downloader/f4m.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 87d3150c7..744bf91cb 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -187,6 +187,10 @@ def build_fragments_list(boot_info):
     return res
 
 
+def write_unsigned_int(stream, val):
+    stream.write(struct_pack('!I', val))
+
+
 def write_flv_header(stream):
     """Writes the FLV header to stream"""
     # FLV header
@@ -198,6 +202,8 @@ def write_flv_header(stream):
 
 def write_metadata_tag(stream, metadata):
     """Writes optional metadata tag to stream"""
+    FLV_TAG_HEADER_LEN = 11
+
     if metadata:
         # Script data
         stream.write(b'\x12')
@@ -205,9 +211,7 @@ def write_metadata_tag(stream, metadata):
         stream.write(struct_pack('!L', len(metadata))[1:])
         stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
         stream.write(metadata)
-        # Magic numbers extracted from the output files produced by AdobeHDS.php
-        # (https://github.com/K-S-V/Scripts)
-        stream.write(b'\x00\x00\x01\x73')
+        write_unsigned_int(stream, FLV_TAG_HEADER_LEN + len(metadata))
 
 
 def _add_ns(prop):

From f14f2a6d7909eae5cf0d6f447f8e8a3eb53752d4 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Mon, 5 Jan 2015 20:30:40 +0200
Subject: [PATCH 417/974] [downloader/f4m] Minor cleanup

---
 youtube_dl/downloader/f4m.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 744bf91cb..c460c167a 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -191,6 +191,10 @@ def write_unsigned_int(stream, val):
     stream.write(struct_pack('!I', val))
 
 
+def write_unsigned_int_24(stream, val):
+    stream.write(struct_pack('!I', val)[1:])
+
+
 def write_flv_header(stream):
     """Writes the FLV header to stream"""
     # FLV header
@@ -202,13 +206,12 @@ def write_flv_header(stream):
 
 def write_metadata_tag(stream, metadata):
     """Writes optional metadata tag to stream"""
+    SCRIPT_TAG = b'\x12'
     FLV_TAG_HEADER_LEN = 11
 
     if metadata:
-        # Script data
-        stream.write(b'\x12')
-        # Size of the metadata with 3 bytes
-        stream.write(struct_pack('!L', len(metadata))[1:])
+        stream.write(SCRIPT_TAG)
+        write_unsigned_int_24(stream, len(metadata))
         stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
         stream.write(metadata)
         write_unsigned_int(stream, FLV_TAG_HEADER_LEN + len(metadata))

From ee61f6f3e2f4d03a3ad9c5ca8f4fc5cd9461bdbc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 22:42:17 +0100
Subject: [PATCH 418/974] [youtube] Handle cases where format comes without a
 preference (Fixes #4648)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d1bbf0b01..4aa8615b4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1046,7 +1046,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     for f in formats:
                         if f['format_id'] in dash_keys:
                             f['format_id'] = 'nondash-%s' % f['format_id']
-                            f['preference'] -= 10000
+                            f['preference'] = f.get('preference', 0) - 10000
                     formats.extend(dash_formats)
 
         self._sort_formats(formats)

From f8bb576c4f376c9514a57386b924e67484f52c36 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 22:42:38 +0100
Subject: [PATCH 419/974] release 2015.01.05.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 086f0ebf0..2124e954f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.05'
+__version__ = '2015.01.05.1'

From 0df2dea73bc311e1b839de7d9b8fe84c8f550718 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Jan 2015 06:54:31 +0600
Subject: [PATCH 420/974] [giga] Add extractor (Closes #4090)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/giga.py     | 101 +++++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/giga.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8e47bd60d..b523e9644 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -159,6 +159,7 @@ from .gametrailers import GametrailersIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
 from .giantbomb import GiantBombIE
+from .giga import GigaIE
 from .glide import GlideIE
 from .globo import GloboIE
 from .godtube import GodTubeIE
diff --git a/youtube_dl/extractor/giga.py b/youtube_dl/extractor/giga.py
new file mode 100644
index 000000000..775890112
--- /dev/null
+++ b/youtube_dl/extractor/giga.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import itertools
+
+from .common import InfoExtractor
+from ..utils import (
+    qualities,
+    compat_str,
+    parse_duration,
+    parse_iso8601,
+    str_to_int,
+)
+
+
+class GigaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?giga\.de/(?:[^/]+/)*(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'http://www.giga.de/filme/anime-awesome/trailer/anime-awesome-chihiros-reise-ins-zauberland-das-beste-kommt-zum-schluss/',
+        'md5': '6bc5535e945e724640664632055a584f',
+        'info_dict': {
+            'id': '2622086',
+            'display_id': 'anime-awesome-chihiros-reise-ins-zauberland-das-beste-kommt-zum-schluss',
+            'ext': 'mp4',
+            'title': 'Anime Awesome: Chihiros Reise ins Zauberland – Das Beste kommt zum Schluss',
+            'description': 'md5:afdf5862241aded4718a30dff6a57baf',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 578,
+            'timestamp': 1414749706,
+            'upload_date': '20141031',
+            'uploader': 'Robin Schweiger',
+            'view_count': int,
+        },
+    }, {
+        'url': 'http://www.giga.de/games/channel/giga-top-montag/giga-topmontag-die-besten-serien-2014/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.giga.de/extra/netzkultur/videos/giga-games-tom-mats-robin-werden-eigene-wege-gehen-eine-ankuendigung/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.giga.de/tv/jonas-liest-spieletitel-eingedeutscht-episode-2/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            [r'data-video-id="(\d+)"', r'/api/video/jwplayer/#v=(\d+)'],
+            webpage, 'video id')
+
+        playlist = self._download_json(
+            'http://www.giga.de/api/syndication/video/video_id/%s/playlist.json?content=syndication/key/368b5f151da4ae05ced7fa296bdff65a/'
+            % video_id, video_id)[0]
+
+        quality = qualities(['normal', 'hd720'])
+
+        formats = []
+        for format_id in itertools.count(0):
+            fmt = playlist.get(compat_str(format_id))
+            if not fmt:
+                break
+            formats.append({
+                'url': fmt['src'],
+                'format_id': '%s-%s' % (fmt['quality'], fmt['type'].split('/')[-1]),
+                'quality': quality(fmt['quality']),
+            })
+        self._sort_formats(formats)
+
+        title = self._html_search_meta(
+            'title', webpage, 'title', fatal=True)
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        duration = parse_duration(self._search_regex(
+            r'(?s)(?:data-video-id="{0}"|data-video="[^"]*/api/video/jwplayer/#v={0}[^"]*")[^>]*>.+?<span class="duration">([^<]+)</span>'.format(video_id),
+            webpage, 'duration', fatal=False))
+
+        timestamp = parse_iso8601(self._search_regex(
+            r'datetime="([^"]+)"', webpage, 'upload date', fatal=False))
+        uploader = self._search_regex(
+            r'class="author">([^<]+)</a>', webpage, 'uploader', fatal=False)
+
+        view_count = str_to_int(self._search_regex(
+            r'<span class="views"><strong>([\d.]+)</strong>', webpage, 'view count', fatal=False))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'view_count': view_count,
+            'formats': formats,
+        }

From 76b3c61012b5cd5a539e2a1a121d11427e6c4c6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 6 Jan 2015 11:59:41 +0100
Subject: [PATCH 421/974] [youtube] Add formats 308 and 315 (closes #4650)

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4aa8615b4..f0efaf0d9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -287,7 +287,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'VP9'},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
 
         # Dash webm audio
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},

From 27a82a1b938f149b8c718e41ae57fcba01af299e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jan 2015 05:03:14 +0600
Subject: [PATCH 422/974] [ceskatelevize] Simplify

---
 youtube_dl/extractor/ceskatelevize.py | 32 +++++++++------------------
 1 file changed, 11 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 345ac5e85..0a9bd01c9 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -12,7 +12,6 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     float_or_none,
-    HEADRequest,
 )
 
 
@@ -105,14 +104,16 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
         duration = float_or_none(item.get('duration'))
         thumbnail = item.get('previewImageUrl')
 
-        # subtitles
-        subtitles = self.extract_subtitles(video_id, webpage)
+        subtitles = {}
+        subs = item.get('subtitles')
+        if subs:
+            subtitles['cs'] = subs[0]['url']
 
         if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, webpage)
+            self._list_available_subtitles(video_id, subtitles)
             return
 
-        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, webpage))
+        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, subtitles))
 
         return {
             'id': episode_id,
@@ -124,13 +125,14 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
             'subtitles': subtitles,
         }
 
-    def _fix_subtitles(self, subtitles):
-        """ Convert milisecond-based subtitles to SRT """
+    @staticmethod
+    def _fix_subtitles(subtitles):
+        """ Convert millisecond-based subtitles to SRT """
         if subtitles is None:
             return subtitles  # subtitles not requested
 
         def _msectotimecode(msec):
-            """ Helper utility to convert miliseconds to timecode """
+            """ Helper utility to convert milliseconds to timecode """
             components = []
             for divider in [1000, 60, 60, 100]:
                 components.append(msec % divider)
@@ -139,7 +141,7 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
 
         def _fix_subtitle(subtitle):
             for line in subtitle.splitlines():
-                m = re.match(r"^ *([0-9]+); *([0-9]+) +([0-9]+) *$", line)
+                m = re.match(r"^\s*([0-9]+);\s*([0-9]+)\s+([0-9]+)\s*$", line)
                 if m:
                     yield m.group(1)
                     start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
@@ -151,15 +153,3 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
         for k, v in subtitles.items():
             fixed_subtitles[k] = "\r\n".join(_fix_subtitle(v))
         return fixed_subtitles
-
-    def _get_available_subtitles(self, video_id, webpage):
-        video_id = video_id.partition('-')[0]
-        url = 'http://imgct.ceskatelevize.cz/cache/data/ivysilani/' \
-              'subtitles/{}/{}/sub.txt'.format(video_id[:3], video_id)
-        req = HEADRequest(url)
-        sub = self._request_webpage(
-            req, video_id,
-            note="Checking subtitles",
-            errnote="No subtitles found",
-            fatal=False)
-        return {'cs': url} if sub else {}

From 6309cb9b412a6e9914bd873201ad0fc38ac2659d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jan 2015 05:03:34 +0600
Subject: [PATCH 423/974] [ceskatelevize] Fix python 2.6 format issue

---
 youtube_dl/extractor/ceskatelevize.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 0a9bd01c9..f70e090bb 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -145,7 +145,7 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
                 if m:
                     yield m.group(1)
                     start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
-                    yield "{} --> {}".format(start, stop)
+                    yield "{0} --> {1}".format(start, stop)
                 else:
                     yield line
 

From 0b54a5b10ab9da3257130b0061fb46c84b4d64ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jan 2015 05:04:15 +0600
Subject: [PATCH 424/974] [ceskatelevize] Add subtitles tests

---
 test/test_subtitles.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index d34565191..6336dd317 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -17,6 +17,7 @@ from youtube_dl.extractor import (
     TEDIE,
     VimeoIE,
     WallaIE,
+    CeskaTelevizeIE,
 )
 
 
@@ -317,5 +318,32 @@ class TestWallaSubtitles(BaseTestSubtitles):
         self.assertEqual(len(subtitles), 0)
 
 
+class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
+    url = 'http://www.ceskatelevize.cz/ivysilani/10600540290-u6-uzasny-svet-techniky'
+    IE = CeskaTelevizeIE
+
+    def test_list_subtitles(self):
+        self.DL.expect_warning('Automatic Captions not supported by this server')
+        self.DL.params['listsubtitles'] = True
+        info_dict = self.getInfoDict()
+        self.assertEqual(info_dict, None)
+
+    def test_allsubtitles(self):
+        self.DL.expect_warning('Automatic Captions not supported by this server')
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['cs']))
+        self.assertEqual(md5(subtitles['cs']), '9bf52d9549533c32c427e264bf0847d4')
+
+    def test_nosubtitles(self):
+        self.DL.expect_warning('video doesn\'t have subtitles')
+        self.url = 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220'
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(len(subtitles), 0)
+
+
 if __name__ == '__main__':
     unittest.main()

From d6a31b17661b6d1a0d3fd987ef9570011e458e6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jan 2015 05:05:18 +0600
Subject: [PATCH 425/974] Credit @oskar456 for ceskatelevize subtitles support
 (#4622)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index a63c97ae0..8f2010803 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -100,3 +100,4 @@ Cédric Luthi
 Thijs Vermeir
 Joel Leclerc
 Christopher Krooss
+Ondřej Caletka

From 799d88d3d8a9337fb0b7a858ec4ffb3aaacbc974 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jan 2015 06:46:56 +0600
Subject: [PATCH 426/974] [nrktv] Add support for playlists (Closes #4656)

---
 youtube_dl/extractor/nrk.py | 128 ++++++++++++++++++++++++++++++------
 1 file changed, 107 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 43e8e619f..e950c76dd 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -72,7 +72,7 @@ class NRKIE(InfoExtractor):
 
 
 class NRKTVIE(InfoExtractor):
-    _VALID_URL = r'http://tv\.nrk(?:super)?\.no/(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})'
+    _VALID_URL = r'http://tv\.nrk(?:super)?\.no/(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
         {
@@ -85,7 +85,7 @@ class NRKTVIE(InfoExtractor):
                 'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
                 'upload_date': '20140523',
                 'duration': 1741.52,
-            }
+            },
         },
         {
             'url': 'http://tv.nrk.no/program/mdfp15000514',
@@ -97,39 +97,125 @@ class NRKTVIE(InfoExtractor):
                 'description': 'md5:654c12511f035aed1e42bdf5db3b206a',
                 'upload_date': '20140524',
                 'duration': 4605.0,
-            }
+            },
         },
+        {
+            # single playlist video
+            'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
+            'md5': 'adbd1dbd813edaf532b0a253780719c2',
+            'info_dict': {
+                'id': 'MSPO40010515-part2',
+                'ext': 'flv',
+                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
+                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                'upload_date': '20150106',
+            },
+            'skip': 'Only works from Norway',
+            'params': {
+                'proxy': '127.0.0.1:8118',
+            },
+        },
+        {
+            'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
+            'playlist': [
+                {
+                    'md5': '9480285eff92d64f06e02a5367970a7a',
+                    'info_dict': {
+                        'id': 'MSPO40010515-part1',
+                        'ext': 'flv',
+                        'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 1:2)',
+                        'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                        'upload_date': '20150106',
+                    },
+                },
+                {
+                    'md5': 'adbd1dbd813edaf532b0a253780719c2',
+                    'info_dict': {
+                        'id': 'MSPO40010515-part2',
+                        'ext': 'flv',
+                        'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
+                        'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                        'upload_date': '20150106',
+                    },
+                },
+            ],
+            'info_dict': {
+                'id': 'MSPO40010515',
+                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn',
+                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                'upload_date': '20150106',
+                'duration': 6947.5199999999995,
+            },
+            'skip': 'Only works from Norway',
+            'params': {
+                'proxy': '127.0.0.1:8118',
+            },
+        }
     ]
 
+    def _extract_f4m(self, manifest_url, video_id):
+        return self._extract_f4m_formats(manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id)
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        part_id = mobj.group('part_id')
 
-        page = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_meta('title', page, 'title')
-        description = self._html_search_meta('description', page, 'description')
-        thumbnail = self._html_search_regex(r'data-posterimage="([^"]+)"', page, 'thumbnail', fatal=False)
-        upload_date = unified_strdate(self._html_search_meta('rightsfrom', page, 'upload date', fatal=False))
-        duration = float_or_none(
-            self._html_search_regex(r'data-duration="([^"]+)"', page, 'duration', fatal=False))
+        title = self._html_search_meta(
+            'title', webpage, 'title')
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+
+        thumbnail = self._html_search_regex(
+            r'data-posterimage="([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
+        upload_date = unified_strdate(self._html_search_meta(
+            'rightsfrom', webpage, 'upload date', fatal=False))
+        duration = float_or_none(self._html_search_regex(
+            r'data-duration="([^"]+)"',
+            webpage, 'duration', fatal=False))
+
+        # playlist
+        parts = re.findall(
+            r'<a href="#del=(\d+)"[^>]+data-argument="([^"]+)">([^<]+)</a>', webpage)
+        if parts:
+            entries = []
+            for current_part_id, stream_url, part_title in parts:
+                if part_id and current_part_id != part_id:
+                    continue
+                video_part_id = '%s-part%s' % (video_id, current_part_id)
+                formats = self._extract_f4m(stream_url, video_part_id)
+                entries.append({
+                    'id': video_part_id,
+                    'title': part_title,
+                    'description': description,
+                    'thumbnail': thumbnail,
+                    'upload_date': upload_date,
+                    'formats': formats,
+                })
+            if part_id:
+                if entries:
+                    return entries[0]
+            else:
+                playlist = self.playlist_result(entries, video_id, title, description)
+                playlist.update({
+                    'thumbnail': thumbnail,
+                    'upload_date': upload_date,
+                    'duration': duration,
+                })
+                return playlist
 
         formats = []
 
-        f4m_url = re.search(r'data-media="([^"]+)"', page)
+        f4m_url = re.search(r'data-media="([^"]+)"', webpage)
         if f4m_url:
-            formats.append({
-                'url': f4m_url.group(1) + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124',
-                'format_id': 'f4m',
-                'ext': 'flv',
-            })
+            formats.extend(self._extract_f4m(f4m_url.group(1), video_id))
 
-        m3u8_url = re.search(r'data-hls-media="([^"]+)"', page)
+        m3u8_url = re.search(r'data-hls-media="([^"]+)"', webpage)
         if m3u8_url:
-            formats.append({
-                'url': m3u8_url.group(1),
-                'format_id': 'm3u8',
-            })
+            formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4'))
 
         self._sort_formats(formats)
 

From 059006292523264f4e7c7e03df3729612af8099c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:20:20 +0100
Subject: [PATCH 427/974] Respect age_limit when listing extractors (Fixes
 #4653)

---
 test/helper.py                   | 14 ++------------
 test/test_utils.py               |  8 ++++++++
 youtube_dl/YoutubeDL.py          | 10 +++-------
 youtube_dl/__init__.py           |  8 +++-----
 youtube_dl/extractor/__init__.py | 13 +++++++++++++
 youtube_dl/extractor/common.py   | 30 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/xtube.py    |  2 ++
 youtube_dl/utils.py              | 10 ++++++++++
 8 files changed, 71 insertions(+), 24 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 96d58b7c1..77225e4f7 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -82,18 +82,8 @@ class FakeYDL(YoutubeDL):
 
 def gettestcases(include_onlymatching=False):
     for ie in youtube_dl.extractor.gen_extractors():
-        t = getattr(ie, '_TEST', None)
-        if t:
-            assert not hasattr(ie, '_TESTS'), \
-                '%s has _TEST and _TESTS' % type(ie).__name__
-            tests = [t]
-        else:
-            tests = getattr(ie, '_TESTS', [])
-        for t in tests:
-            if not include_onlymatching and t.get('only_matching', False):
-                continue
-            t['name'] = type(ie).__name__[:-len('IE')]
-            yield t
+        for tc in ie.get_testcases(include_onlymatching):
+            yield tc
 
 
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
diff --git a/test/test_utils.py b/test/test_utils.py
index dd49a6d17..16e1a1ddf 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -16,6 +16,7 @@ import json
 import xml.etree.ElementTree
 
 from youtube_dl.utils import (
+    age_restricted,
     args_to_str,
     clean_html,
     DateRange,
@@ -402,5 +403,12 @@ Trying to open render node...
 Success at /dev/dri/renderD128.
 ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
 
+    def test_age_restricted(self):
+        self.assertFalse(age_restricted(None, 10))  # unrestricted content
+        self.assertFalse(age_restricted(1, None))  # unrestricted policy
+        self.assertFalse(age_restricted(8, 10))
+        self.assertTrue(age_restricted(18, 14))
+        self.assertFalse(age_restricted(18, 18))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 806e7b239..36c71f947 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -63,6 +63,7 @@ from .utils import (
     YoutubeDLHandler,
     prepend_extension,
     args_to_str,
+    age_restricted,
 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors
@@ -550,13 +551,8 @@ class YoutubeDL(object):
             max_views = self.params.get('max_views')
             if max_views is not None and view_count > max_views:
                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
-        age_limit = self.params.get('age_limit')
-        if age_limit is not None:
-            actual_age_limit = info_dict.get('age_limit')
-            if actual_age_limit is None:
-                actual_age_limit = 0
-            if age_limit < actual_age_limit:
-                return 'Skipping "' + title + '" because it is age restricted'
+        if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
+            return 'Skipping "%s" because it is age restricted' % title
         if self.in_download_archive(info_dict):
             return '%s has already been recorded in archive' % video_title
         return None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 56f560d26..4c21188a9 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -38,7 +38,7 @@ from .update import update_self
 from .downloader import (
     FileDownloader,
 )
-from .extractor import gen_extractors
+from .extractor import list_extractors
 from .YoutubeDL import YoutubeDL
 
 
@@ -95,17 +95,15 @@ def _real_main(argv=None):
     _enc = preferredencoding()
     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 
-    extractors = gen_extractors()
-
     if opts.list_extractors:
-        for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
+        for ie in list_extractors(opts.age_limit):
             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
             matchedUrls = [url for url in all_urls if ie.suitable(url)]
             for mu in matchedUrls:
                 compat_print('  ' + mu)
         sys.exit(0)
     if opts.list_extractor_descriptions:
-        for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
+        for ie in list_extractors(opts.age_limit):
             if not ie._WORKING:
                 continue
             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b523e9644..0145e350d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -560,6 +560,8 @@ from .zingmp3 import (
     ZingMp3AlbumIE,
 )
 
+from ..utils import age_restricted
+
 _ALL_CLASSES = [
     klass
     for name, klass in globals().items()
@@ -575,6 +577,17 @@ def gen_extractors():
     return [klass() for klass in _ALL_CLASSES]
 
 
+def list_extractors(age_limit):
+    """
+    Return a list of extractors that are suitable for the given age,
+    sorted by extractor ID.
+    """
+
+    return sorted(
+        filter(lambda ie: ie.is_suitable(age_limit), gen_extractors()),
+        key=lambda ie: ie.IE_NAME.lower())
+
+
 def get_info_extractor(ie_name):
     """Returns the info extractor class with the given ie_name"""
     return globals()[ie_name + 'IE']
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 562e656e0..df32b5ca0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -21,6 +21,7 @@ from ..compat import (
     compat_str,
 )
 from ..utils import (
+    age_restricted,
     clean_html,
     compiled_regex_type,
     ExtractorError,
@@ -877,6 +878,35 @@ class InfoExtractor(object):
             None, '/', True, False, expire_time, '', None, None, None)
         self._downloader.cookiejar.set_cookie(cookie)
 
+    def get_testcases(self, include_onlymatching=False):
+        t = getattr(self, '_TEST', None)
+        if t:
+            assert not hasattr(self, '_TESTS'), \
+                '%s has _TEST and _TESTS' % type(self).__name__
+            tests = [t]
+        else:
+            tests = getattr(self, '_TESTS', [])
+        for t in tests:
+            if not include_onlymatching and t.get('only_matching', False):
+                continue
+            t['name'] = type(self).__name__[:-len('IE')]
+            yield t
+
+    def is_suitable(self, age_limit):
+        """ Test whether the extractor is generally suitable for the given
+        age limit (i.e. pornographic sites are not, all others usually are) """
+
+        any_restricted = False
+        for tc in self.get_testcases(include_onlymatching=False):
+            if 'playlist' in tc:
+                tc = tc['playlist'][0]
+            is_restricted = age_restricted(
+                tc.get('info_dict', {}).get('age_limit'), age_limit)
+            if not is_restricted:
+                return True
+            any_restricted = any_restricted or is_restricted
+        return not any_restricted
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 95f1c8f3c..e8490b028 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -95,6 +95,7 @@ class XTubeUserIE(InfoExtractor):
         'url': 'http://www.xtube.com/community/profile.php?user=greenshowers',
         'info_dict': {
             'id': 'greenshowers',
+            'age_limit': 18,
         },
         'playlist_mincount': 155,
     }
@@ -124,6 +125,7 @@ class XTubeUserIE(InfoExtractor):
         return {
             '_type': 'playlist',
             'id': username,
+            'age_limit': 18,
             'entries': [{
                 '_type': 'url',
                 'url': eurl,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d4951c406..29739a483 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1560,3 +1560,13 @@ def urlhandle_detect_ext(url_handle):
         getheader = url_handle.info().getheader
 
     return getheader('Content-Type').split("/")[1]
+
+
+def age_restricted(content_limit, age_limit):
+    """ Returns True iff the content should be blocked """
+
+    if age_limit is None:  # No limit set
+        return False
+    if content_limit is None:
+        return False  # Content available for everyone
+    return age_limit < content_limit

From 8ee341500dea885c79316fb8f12adde2028c55a5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:21:24 +0100
Subject: [PATCH 428/974] [viki] Modernize

---
 youtube_dl/extractor/viki.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 15f315298..944901e14 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -17,7 +17,6 @@ class VikiIE(SubtitlesInfoExtractor):
     _VALID_URL = r'^https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
     _TEST = {
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
-        'md5': 'a21454021c2646f5433514177e2caa5f',
         'info_dict': {
             'id': '1023585v',
             'ext': 'mp4',
@@ -31,8 +30,7 @@ class VikiIE(SubtitlesInfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         title = self._og_search_title(webpage)

From 75e51819d010b38d9a3464f8749688c8d53fc123 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:22:28 +0100
Subject: [PATCH 429/974] release 2015.01.07

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2124e954f..f9c934351 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.05.1'
+__version__ = '2015.01.07'

From fd86c2026de4ca071d1f6bd432da68c93fcf184e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:31:38 +0100
Subject: [PATCH 430/974] release 2015.01.07.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f9c934351..e20525b58 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.07'
+__version__ = '2015.01.07.1'

From ccc5842bc9d96abaf6e54b91101f7e281b05663c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:37:21 +0100
Subject: [PATCH 431/974] [gameone] Modernize

---
 youtube_dl/extractor/gameone.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gameone.py b/youtube_dl/extractor/gameone.py
index 75f180928..a07d69841 100644
--- a/youtube_dl/extractor/gameone.py
+++ b/youtube_dl/extractor/gameone.py
@@ -57,8 +57,7 @@ class GameOneIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         og_video = self._og_search_video_url(webpage, secure=False)

From 2bad0e5d202aa041ab9ddb9e7dcffe69848a5349 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:41:05 +0100
Subject: [PATCH 432/974] [/__init__] Define public API

---
 devscripts/gh-pages/update-sites.py | 2 +-
 youtube_dl/__init__.py              | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py
index f0f0481c7..d3ef5f0b5 100755
--- a/devscripts/gh-pages/update-sites.py
+++ b/devscripts/gh-pages/update-sites.py
@@ -16,7 +16,7 @@ def main():
         template = tmplf.read()
 
     ie_htmls = []
-    for ie in sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower()):
+    for ie in youtube_dl.list_extractors(age_limit=None):
         ie_html = '<b>{}</b>'.format(ie.IE_NAME)
         ie_desc = getattr(ie, 'IE_DESC', None)
         if ie_desc is False:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 4c21188a9..4aa7fba6a 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -38,7 +38,7 @@ from .update import update_self
 from .downloader import (
     FileDownloader,
 )
-from .extractor import list_extractors
+from .extractor import gen_extractors, list_extractors
 from .YoutubeDL import YoutubeDL
 
 
@@ -363,3 +363,5 @@ def main(argv=None):
         sys.exit('ERROR: fixed output name but more than one file to download')
     except KeyboardInterrupt:
         sys.exit('\nERROR: Interrupted by user')
+
+__all__ = ['main', 'YoutubeDL', 'gen_extractors', 'list_extractors']

From 900813a328813ae9095ebfd2b2b846a9d98e2821 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:41:48 +0100
Subject: [PATCH 433/974] release 2015.01.07.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e20525b58..164ce0268 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.07.1'
+__version__ = '2015.01.07.2'

From c758bf9fd7625b8db3c64a06e100ddc5a4ae07d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 7 Jan 2015 11:56:22 +0100
Subject: [PATCH 434/974] [nrktv] Remove 'proxy' parameter from tests

---
 youtube_dl/extractor/nrk.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index e950c76dd..321ce5ce7 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -111,9 +111,6 @@ class NRKTVIE(InfoExtractor):
                 'upload_date': '20150106',
             },
             'skip': 'Only works from Norway',
-            'params': {
-                'proxy': '127.0.0.1:8118',
-            },
         },
         {
             'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
@@ -147,9 +144,6 @@ class NRKTVIE(InfoExtractor):
                 'duration': 6947.5199999999995,
             },
             'skip': 'Only works from Norway',
-            'params': {
-                'proxy': '127.0.0.1:8118',
-            },
         }
     ]
 

From 5d8993b06adb24b9c275a42a67dc6bbeae590407 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 7 Jan 2015 11:59:15 +0100
Subject: [PATCH 435/974] [extractor/__init__] Remove unused import

---
 youtube_dl/extractor/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0145e350d..8dacc2c54 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -560,8 +560,6 @@ from .zingmp3 import (
     ZingMp3AlbumIE,
 )
 
-from ..utils import age_restricted
-
 _ALL_CLASSES = [
     klass
     for name, klass in globals().items()

From c5d666d3741d076d668c16f0dd3601589484b7c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 7 Jan 2015 16:09:43 +0100
Subject: [PATCH 436/974] Fix build with python 2.6

* Packages cannot be executed
* '.format' needs the index of the argument

(Reported in https://github.com/Homebrew/homebrew/issues/35616)
---
 Makefile                          | 2 +-
 devscripts/make_supportedsites.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index e53a367ef..578079879 100644
--- a/Makefile
+++ b/Makefile
@@ -63,7 +63,7 @@ youtube-dl: youtube_dl/*.py youtube_dl/*/*.py
 	chmod a+x youtube-dl
 
 README.md: youtube_dl/*.py youtube_dl/*/*.py
-	COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py
+	COLUMNS=80 python youtube_dl/__main__.py --help | python devscripts/make_readme.py
 
 CONTRIBUTING.md: README.md
 	python devscripts/make_contributing.py README.md CONTRIBUTING.md
diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py
index 140010644..3df4385a6 100644
--- a/devscripts/make_supportedsites.py
+++ b/devscripts/make_supportedsites.py
@@ -23,12 +23,12 @@ def main():
 
     def gen_ies_md(ies):
         for ie in ies:
-            ie_md = '**{}**'.format(ie.IE_NAME)
+            ie_md = '**{0}**'.format(ie.IE_NAME)
             ie_desc = getattr(ie, 'IE_DESC', None)
             if ie_desc is False:
                 continue
             if ie_desc is not None:
-                ie_md += ': {}'.format(ie.IE_DESC)
+                ie_md += ': {0}'.format(ie.IE_DESC)
             if not ie.working():
                 ie_md += ' (Currently broken)'
             yield ie_md

From 58a84b8cb69e967dd95fa309c0160838853c7d79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jan 2015 01:33:22 +0600
Subject: [PATCH 437/974] [bilibili] Fix extraction (Closes #4660)

---
 youtube_dl/extractor/bilibili.py | 67 ++++++++++++++------------------
 1 file changed, 29 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 241b904a9..75d744852 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -4,9 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_parse_qs
 from ..utils import (
-    ExtractorError,
     int_or_none,
     unified_strdate,
 )
@@ -54,45 +52,38 @@ class BiliBiliIE(InfoExtractor):
         thumbnail = self._html_search_meta(
             'thumbnailUrl', video_code, 'thumbnail', fatal=False)
 
-        player_params = compat_parse_qs(self._html_search_regex(
-            r'<iframe .*?class="player" src="https://secure\.bilibili\.(?:tv|com)/secure,([^"]+)"',
-            webpage, 'player params'))
+        cid = self._search_regex(r'cid=(\d+)', webpage, 'cid')
 
-        if 'cid' in player_params:
-            cid = player_params['cid'][0]
+        lq_doc = self._download_xml(
+            'http://interface.bilibili.com/v_cdn_play?appkey=1&cid=%s' % cid,
+            video_id,
+            note='Downloading LQ video info'
+        )
+        lq_durl = lq_doc.find('./durl')
+        formats = [{
+            'format_id': 'lq',
+            'quality': 1,
+            'url': lq_durl.find('./url').text,
+            'filesize': int_or_none(
+                lq_durl.find('./size'), get_attr='text'),
+        }]
 
-            lq_doc = self._download_xml(
-                'http://interface.bilibili.cn/v_cdn_play?cid=%s' % cid,
-                video_id,
-                note='Downloading LQ video info'
-            )
-            lq_durl = lq_doc.find('.//durl')
-            formats = [{
-                'format_id': 'lq',
-                'quality': 1,
-                'url': lq_durl.find('./url').text,
+        hq_doc = self._download_xml(
+            'http://interface.bilibili.com/playurl?appkey=1&cid=%s' % cid,
+            video_id,
+            note='Downloading HQ video info',
+            fatal=False,
+        )
+        if hq_doc is not False:
+            hq_durl = hq_doc.find('./durl')
+            formats.append({
+                'format_id': 'hq',
+                'quality': 2,
+                'ext': 'flv',
+                'url': hq_durl.find('./url').text,
                 'filesize': int_or_none(
-                    lq_durl.find('./size'), get_attr='text'),
-            }]
-
-            hq_doc = self._download_xml(
-                'http://interface.bilibili.cn/playurl?cid=%s' % cid,
-                video_id,
-                note='Downloading HQ video info',
-                fatal=False,
-            )
-            if hq_doc is not False:
-                hq_durl = hq_doc.find('.//durl')
-                formats.append({
-                    'format_id': 'hq',
-                    'quality': 2,
-                    'ext': 'flv',
-                    'url': hq_durl.find('./url').text,
-                    'filesize': int_or_none(
-                        hq_durl.find('./size'), get_attr='text'),
-                })
-        else:
-            raise ExtractorError('Unsupported player parameters: %r' % (player_params,))
+                    hq_durl.find('./size'), get_attr='text'),
+            })
 
         self._sort_formats(formats)
         return {

From fdd6e18b75ef580ff9097f8589e38dcce3a7fa3a Mon Sep 17 00:00:00 2001
From: Kieran O'Reilly <kieranoreilly@college.harvard.edu>
Date: Wed, 7 Jan 2015 20:23:36 -0800
Subject: [PATCH 438/974] Spelling

Corrected the spelling of incapacitated
---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0ff7b395a..7917abfc6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -44,7 +44,7 @@ In particular, every site support request issue should only pertain to services
 
 ###  Is anyone going to need the feature?
 
-Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
+Only post features that you (or an incapacitated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
 
 ###  Is your question about youtube-dl?
 

From 52fc3ba405cf9d87a4ada50acded5dbe10eca492 Mon Sep 17 00:00:00 2001
From: Li4ick <danielciugurean@yahoo.com>
Date: Thu, 8 Jan 2015 14:17:26 +0200
Subject: [PATCH 439/974] Change path name to MSDN standard.

<Yourname> changed to <user name>, which is more general.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 915bcd0cd..1df4fe2f1 100644
--- a/README.md
+++ b/README.md
@@ -326,7 +326,7 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<Yourname>\youtube-dl.conf`.
+You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<user name>\youtube-dl.conf`.
 
 # OUTPUT TEMPLATE
 

From 398133cf554895859cf587654e67049d2e21b570 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jan 2015 19:07:28 +0600
Subject: [PATCH 440/974] [huffpost] Make extraction more robust (Closes #4663)

---
 youtube_dl/extractor/huffpost.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/huffpost.py b/youtube_dl/extractor/huffpost.py
index 4ccf6b9b8..a38eae421 100644
--- a/youtube_dl/extractor/huffpost.py
+++ b/youtube_dl/extractor/huffpost.py
@@ -39,8 +39,9 @@ class HuffPostIE(InfoExtractor):
         data = self._download_json(api_url, video_id)['data']
 
         video_title = data['title']
-        duration = parse_duration(data['running_time'])
-        upload_date = unified_strdate(data['schedule']['starts_at'])
+        duration = parse_duration(data.get('running_time'))
+        upload_date = unified_strdate(
+            data.get('schedule', {}).get('starts_at') or data.get('segment_start_date_time'))
         description = data.get('description')
 
         thumbnails = []
@@ -59,16 +60,11 @@ class HuffPostIE(InfoExtractor):
             'ext': 'mp4',
             'url': url,
             'vcodec': 'none' if key.startswith('audio/') else None,
-        } for key, url in data['sources']['live'].items()]
-        if data.get('fivemin_id'):
-            fid = data['fivemin_id']
-            fcat = str(int(fid) // 100 + 1)
-            furl = 'http://avideos.5min.com/2/' + fcat[-3:] + '/' + fcat + '/' + fid + '.mp4'
-            formats.append({
-                'format': 'fivemin',
-                'url': furl,
-                'preference': 1,
-            })
+        } for key, url in data.get('sources', {}).get('live', {}).items()]
+
+        if not formats and data.get('fivemin_id'):
+            return self.url_result('5min:%s' % data['fivemin_id'])
+
         self._sort_formats(formats)
 
         return {

From 99673f04bc5b179241d121cf2aa50f25b6b1c7d0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:21:40 +0100
Subject: [PATCH 441/974] [washingtonpost] Modernize and correct test case

---
 youtube_dl/extractor/washingtonpost.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index 88bbbb219..c17bebd6e 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -10,14 +10,14 @@ from ..utils import (
 
 
 class WashingtonPostIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
     _TEST = {
         'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
         'info_dict': {
             'title': 'Sinkhole of bureaucracy',
         },
         'playlist': [{
-            'md5': 'c3f4b4922ffa259243f68e928db2db8c',
+            'md5': '79132cc09ec5309fa590ae46e4cc31bc',
             'info_dict': {
                 'id': 'fc433c38-b146-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
@@ -29,7 +29,7 @@ class WashingtonPostIE(InfoExtractor):
                 'upload_date': '20140322',
             },
         }, {
-            'md5': 'f645a07652c2950cd9134bb852c5f5eb',
+            'md5': 'e1d5734c06865cc504ad99dc2de0d443',
             'info_dict': {
                 'id': '41255e28-b14a-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
@@ -44,10 +44,9 @@ class WashingtonPostIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        page_id = mobj.group('id')
-
+        page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
+
         title = self._og_search_title(webpage)
         uuids = re.findall(r'data-video-uuid="([^"]+)"', webpage)
         entries = []

From adb1307b9aeae4f011ba454a5d64d343e23503c8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:34:27 +0100
Subject: [PATCH 442/974] [imdb] Remove test md5

They seem to reencode quite frequently, so simply remove the md5 sum.
---
 youtube_dl/extractor/imdb.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 13a53a0cb..f29df36b5 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -16,7 +16,6 @@ class ImdbIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.imdb.com/video/imdb/vi2524815897',
-        'md5': '9f34fa777ade3a6e57a054fdbcb3a068',
         'info_dict': {
             'id': '2524815897',
             'ext': 'mp4',

From 211503c39f5ce6e875d18545007060c2764f6f7e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:43:20 +0100
Subject: [PATCH 443/974] [teachertube] Modernize

---
 youtube_dl/extractor/teachertube.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teachertube.py b/youtube_dl/extractor/teachertube.py
index 6c3445d79..82675431f 100644
--- a/youtube_dl/extractor/teachertube.py
+++ b/youtube_dl/extractor/teachertube.py
@@ -57,9 +57,7 @@ class TeacherTubeIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_meta('title', webpage, 'title', fatal=True)

From bec2248141c9cc3e44794d7ee48a6bcfc4904eac Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:43:36 +0100
Subject: [PATCH 444/974] [InfoExtractor/common] Correct and test meta tag
 matching

---
 test/test_InfoExtractor.py     | 18 ++++++++++++++++++
 youtube_dl/extractor/common.py |  2 +-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 13c18ed95..be8d12997 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -40,5 +40,23 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
 
+    def test_html_search_meta(self):
+        ie = self.ie
+        html = '''
+            <meta name="a" content="1" />
+            <meta name='b' content='2'>
+            <meta name="c" content='3'>
+            <meta name=d content='4'>
+            <meta property="e" content='5' >
+            <meta content="6" name="f">
+        '''
+
+        self.assertEqual(ie._html_search_meta('a', html), '1')
+        self.assertEqual(ie._html_search_meta('b', html), '2')
+        self.assertEqual(ie._html_search_meta('c', html), '3')
+        self.assertEqual(ie._html_search_meta('d', html), '4')
+        self.assertEqual(ie._html_search_meta('e', html), '5')
+        self.assertEqual(ie._html_search_meta('f', html), '6')
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index df32b5ca0..d703893dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -594,7 +594,7 @@ class InfoExtractor(object):
         return self._html_search_regex(
             r'''(?isx)<meta
                     (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
-                    [^>]+content=(["\'])(?P<content>.*?)\1''' % re.escape(name),
+                    [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(name),
             html, display_name, fatal=fatal, group='content', **kwargs)
 
     def _dc_search_uploader(self, html):

From 5b78caca94b8913b224222fb94a81481cf8637e9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:45:19 +0100
Subject: [PATCH 445/974] [mit] Amend test definitions

---
 youtube_dl/extractor/mit.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 78787e8f1..3c61a850f 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -105,6 +105,9 @@ class OCWMITIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Lecture 7: Multiple Discrete Random Variables: Expectations, Conditioning, Independence',
                 'description': 'In this lecture, the professor discussed multiple random variables, expectations, and binomial distribution.',
+                'upload_date': '20121109',
+                'uploader_id': 'MIT',
+                'uploader': 'MIT OpenCourseWare',
                 # 'subtitles': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/MIT6_041F11_lec07_300k.mp4.srt'
             }
         },
@@ -114,6 +117,9 @@ class OCWMITIE(InfoExtractor):
                 'id': '7K1sB05pE0A',
                 'ext': 'mp4',
                 'title': 'Session 1: Introduction to Derivatives',
+                'upload_date': '20090818',
+                'uploader_id': 'MIT',
+                'uploader': 'MIT OpenCourseWare',
                 'description': 'This section contains lecture video excerpts, lecture notes, an interactive mathlet with supporting documents, and problem solving videos.',
                 # 'subtitles': 'http://ocw.mit.edu//courses/mathematics/18-01sc-single-variable-calculus-fall-2010/ocw-18.01-f07-lec01_300k.SRT'
             }

From 796858a53f3bb987e47ebe28fb0c1963703541cb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:47:03 +0100
Subject: [PATCH 446/974] [sexykarma] Add age_limit designation

---
 youtube_dl/extractor/sexykarma.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
index c833fc8ee..6446d26dc 100644
--- a/youtube_dl/extractor/sexykarma.py
+++ b/youtube_dl/extractor/sexykarma.py
@@ -24,7 +24,7 @@ class SexyKarmaIE(InfoExtractor):
             'title': 'Taking a quick pee.',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'wildginger7',
-            'upload_date': '20141007',
+            'upload_date': '20141008',
             'duration': 22,
             'view_count': int,
             'comment_count': int,
@@ -45,6 +45,7 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }, {
         'url': 'http://www.watchindianporn.net/video/desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number-dW2mtctxJfs.html',
@@ -61,6 +62,7 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }]
 
@@ -114,4 +116,5 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': view_count,
             'comment_count': comment_count,
             'categories': categories,
+            'age_limit': 18,
         }

From ec3a6a31378925c72facbed1005499780b2d78da Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 14:09:43 +0100
Subject: [PATCH 447/974] [tunein] Ignore reliability if it's >90% (#4097)

---
 youtube_dl/extractor/tunein.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tunein.py b/youtube_dl/extractor/tunein.py
index 4ce5aeeba..b6b1f2568 100644
--- a/youtube_dl/extractor/tunein.py
+++ b/youtube_dl/extractor/tunein.py
@@ -24,7 +24,7 @@ class TuneInIE(InfoExtractor):
     _INFO_DICT = {
         'id': '34682',
         'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
-        'ext': 'AAC',
+        'ext': 'aac',
         'thumbnail': 're:^https?://.*\.png$',
         'location': 'Tacoma, WA',
     }
@@ -78,14 +78,21 @@ class TuneInIE(InfoExtractor):
         for stream in streams:
             if stream.get('Type') == 'Live':
                 is_live = True
+            reliability = stream.get('Reliability')
+            format_note = (
+                'Reliability: %d%%' % reliability
+                if reliability is not None else None)
             formats.append({
+                'preference': (
+                    0 if reliability is None or reliability > 90
+                    else 1),
                 'abr': stream.get('Bandwidth'),
-                'ext': stream.get('MediaType'),
+                'ext': stream.get('MediaType').lower(),
                 'acodec': stream.get('MediaType'),
                 'vcodec': 'none',
                 'url': stream.get('Url'),
-                # Sometimes streams with the highest quality do not exist
-                'preference': stream.get('Reliability'),
+                'source_preference': reliability,
+                'format_note': format_note,
             })
         self._sort_formats(formats)
 

From 9ed99402f5732f84d402c2b098ab20e9ebbea549 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 14:27:01 +0100
Subject: [PATCH 448/974] [youtube] Fix test case

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f0efaf0d9..e71956071 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -414,7 +414,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'id': 'HtVdAasjOgU',
                 'ext': 'mp4',
                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
-                'description': 'md5:eca57043abae25130f58f655ad9a7771',
+                'description': 're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
                 'uploader': 'The Witcher',
                 'uploader_id': 'WitcherGame',
                 'upload_date': '20140605',

From d51a853d5cc4aa17a1c6d60a986387bf242449af Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 14:29:12 +0100
Subject: [PATCH 449/974] [zdf] Fix test case

---
 youtube_dl/extractor/zdf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 74c76a9a0..98f15177b 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -119,7 +119,7 @@ class ZDFChannelIE(InfoExtractor):
         'info_dict': {
             'id': '1586442',
         },
-        'playlist_count': 4,
+        'playlist_count': 3,
     }
     _PAGE_SIZE = 50
 

From bd6b25ce0e5fabd3cb20651a1a3ed8a9cf8748e4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 16:03:13 +0100
Subject: [PATCH 450/974] [fktv] Fix download URL

---
 youtube_dl/extractor/fktv.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index d09d1c13a..190d9f9ad 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 class FKTVIE(InfoExtractor):
     IE_NAME = 'fernsehkritik.tv'
-    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/folge-(?P<ep>[0-9]+)(?:/.*)?'
+    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/folge-(?P<id>[0-9]+)(?:/.*)?'
 
     _TEST = {
         'url': 'http://fernsehkritik.tv/folge-1',
@@ -26,29 +26,32 @@ class FKTVIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        episode = int(mobj.group('ep'))
+        episode = int(self._match_id(url))
 
-        server = random.randint(2, 4)
-        video_thumbnail = 'http://fernsehkritik.tv/images/magazin/folge%d.jpg' % episode
-        start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%d/Start' % episode,
+        video_thumbnail = 'http://fernsehkritik.tv/images/magazin/folge%s.jpg' % episode
+        start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/Start' % episode,
                                                episode)
         playlist = self._search_regex(r'playlist = (\[.*?\]);', start_webpage,
                                       'playlist', flags=re.DOTALL)
         files = json.loads(re.sub('{[^{}]*?}', '{}', playlist))
-        # TODO: return a single multipart video
+
         videos = []
         for i, _ in enumerate(files, 1):
             video_id = '%04d%d' % (episode, i)
-            video_url = 'http://dl%d.fernsehkritik.tv/fernsehkritik%d%s.flv' % (server, episode, '' if i == 1 else '-%d' % i)
+            video_url = 'http://fernsehkritik.tv/js/directme.php?file=%s%s.flv' % (episode, '' if i == 1 else '-%d' % i)
             videos.append({
+                'ext': 'flv',
                 'id': video_id,
                 'url': video_url,
                 'title': clean_html(get_element_by_id('eptitle', start_webpage)),
                 'description': clean_html(get_element_by_id('contentlist', start_webpage)),
                 'thumbnail': video_thumbnail
             })
-        return videos
+        return {
+            '_type': 'multi_video',
+            'entries': videos,
+            'id': 'folge-%s' % episode,
+        }
 
 
 class FKTVPosteckeIE(InfoExtractor):

From 0217c783771a6c8543046869e934f4ced56febc7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 16:14:16 +0100
Subject: [PATCH 451/974] [YoutubeDL] Allow selection by more extensions

---
 test/test_YoutubeDL.py  | 33 +++++++++++++++++++++++++++++++++
 youtube_dl/YoutubeDL.py |  2 +-
 youtube_dl/options.py   | 10 ++++++----
 3 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 730f7ec26..85d87f2c3 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -8,6 +8,8 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import copy
+
 from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
 from youtube_dl.extractor import YoutubeIE
@@ -192,6 +194,37 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'vid-high')
 
+    def test_format_selection_audio_exts(self):
+        formats = [
+            {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
+        ]
+
+        info_dict = _make_result(formats)
+        ydl = YDL({'format': 'best'})
+        ie = YoutubeIE(ydl)
+        ie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(copy.deepcopy(info_dict))
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'aac-64')
+
+        ydl = YDL({'format': 'mp3'})
+        ie = YoutubeIE(ydl)
+        ie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(copy.deepcopy(info_dict))
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'mp3-64')
+
+        ydl = YDL({'prefer_free_formats': True})
+        ie = YoutubeIE(ydl)
+        ie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(copy.deepcopy(info_dict))
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'ogg-64')
+
     def test_format_selection_video(self):
         formats = [
             {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': '_'},
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 36c71f947..24d6c2de7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -786,7 +786,7 @@ class YoutubeDL(object):
             if video_formats:
                 return video_formats[0]
         else:
-            extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a']
+            extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
             if format_spec in extensions:
                 filter_f = lambda f: f['ext'] == format_spec
             else:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a018c1d71..058342dec 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -267,10 +267,12 @@ def parseOpts(overrideArguments=None):
         action='store', dest='format', metavar='FORMAT', default=None,
         help=(
             'video format code, specify the order of preference using'
-            ' slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also'
-            ' supported. You can also use the special names "best",'
-            ' "bestvideo", "bestaudio", "worst", "worstvideo" and'
-            ' "worstaudio". By default, youtube-dl will pick the best quality.'
+            ' slashes, as in -f 22/17/18 . '
+            ' Instead of format codes, you can select by extension for the '
+            'extensions aac, m4a, mp3, mp4, ogg, wav, webm. '
+            'You can also use the special names "best",'
+            ' "bestvideo", "bestaudio", "worst". '
+            ' By default, youtube-dl will pick the best quality.'
             ' Use commas to download multiple audio formats, such as'
             ' -f  136/137/mp4/bestvideo,140/m4a/bestaudio.'
             ' You can merge the video and audio of two formats into a single'

From 11497d5bba4a8a99d5a1907d2a34fe7be7e04c37 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 16:15:08 +0100
Subject: [PATCH 452/974] release 2015.01.08

---
 README.md             | 17 +++++++++--------
 youtube_dl/version.py |  2 +-
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 1df4fe2f1..24b4d7970 100644
--- a/README.md
+++ b/README.md
@@ -248,14 +248,15 @@ which means you can modify it, redistribute it or use it however you like.
 
 ## Video Format Options:
     -f, --format FORMAT              video format code, specify the order of
-                                     preference using slashes: -f 22/17/18 .  -f
-                                     mp4 , -f m4a and  -f flv  are also
-                                     supported. You can also use the special
-                                     names "best", "bestvideo", "bestaudio",
-                                     "worst", "worstvideo" and "worstaudio". By
-                                     default, youtube-dl will pick the best
-                                     quality. Use commas to download multiple
-                                     audio formats, such as -f
+                                     preference using slashes, as in -f 22/17/18
+                                     .  Instead of format codes, you can select
+                                     by extension for the extensions aac, m4a,
+                                     mp3, mp4, ogg, wav, webm. You can also use
+                                     the special names "best", "bestvideo",
+                                     "bestaudio", "worst".  By default, youtube-
+                                     dl will pick the best quality. Use commas
+                                     to download multiple audio formats, such as
+                                     -f
                                      136/137/mp4/bestvideo,140/m4a/bestaudio.
                                      You can merge the video and audio of two
                                      formats into a single file using -f <video-
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 164ce0268..32019e362 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.07.2'
+__version__ = '2015.01.08'

From 811cacdc2c62bea63c6fbd4a3e570744257c37a6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:02:37 +0100
Subject: [PATCH 453/974] [bet] Correct test IDs

---
 youtube_dl/extractor/bet.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index 4fbdd6f1c..d2abd4d77 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -16,7 +16,7 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
             'info_dict': {
-                'id': '406429c6-1b8a-463e-83fc-814adb81a9db',
+                'id': '740ab250-bb94-4a8a-8787-fe0de7c74471',
                 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
                 'ext': 'flv',
                 'title': 'BET News Presents: A Conversation With President Obama',
@@ -35,7 +35,7 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
             'info_dict': {
-                'id': '4160e53b-ad41-43b1-980f-8d85f63121f4',
+                'id': 'bcd1b1df-673a-42cf-8d01-b282db608f2d',
                 'display_id': 'justice-for-ferguson-a-community-reacts',
                 'ext': 'flv',
                 'title': 'Justice for Ferguson: A Community Reacts',
@@ -55,7 +55,6 @@ class BetIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-
         webpage = self._download_webpage(url, display_id)
 
         media_url = compat_urllib_parse.unquote(self._search_regex(

From db6e6250050d011b801f01a665e644bb41be40cf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:05:46 +0100
Subject: [PATCH 454/974] [buzzfeed] Fix test

---
 youtube_dl/extractor/buzzfeed.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/buzzfeed.py b/youtube_dl/extractor/buzzfeed.py
index a40a1bbc4..a5d2af174 100644
--- a/youtube_dl/extractor/buzzfeed.py
+++ b/youtube_dl/extractor/buzzfeed.py
@@ -33,7 +33,7 @@ class BuzzFeedIE(InfoExtractor):
             'skip_download': True,  # Got enough YouTube download tests
         },
         'info_dict': {
-            'description': 'Munchkin the Teddy Bear is back !',
+            'description': 're:Munchkin the Teddy Bear is back ?!',
             'title': 'You Need To Stop What You\'re Doing And Watching This Dog Walk On A Treadmill',
         },
         'playlist': [{
@@ -42,9 +42,9 @@ class BuzzFeedIE(InfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20141124',
                 'uploader_id': 'CindysMunchkin',
-                'description': '© 2014 Munchkin the Shih Tzu\nAll rights reserved\nFacebook: http://facebook.com/MunchkintheShihTzu',
+                'description': 're:© 2014 Munchkin the Shih Tzu',
                 'uploader': 'Munchkin the Shih Tzu',
-                'title': 'Munchkin the Teddy Bear gets her exercise',
+                'title': 're:Munchkin the Teddy Bear gets her exercise',
             },
         }]
     }]

From 5fbf25a681c2f7f6f3470e783fcac65b7c7c339d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:09:28 +0100
Subject: [PATCH 455/974] [test_age_restriction] remove misbehaving test

We now test for the age_limit being set right in test_download, so we don't need more than two tests for the actual age limit handling.
---
 test/test_age_restriction.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py
index 5be065c43..6f5513faa 100644
--- a/test/test_age_restriction.py
+++ b/test/test_age_restriction.py
@@ -45,11 +45,6 @@ class TestAgeRestriction(unittest.TestCase):
             'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
             '505835.mp4', 2, old_age=25)
 
-    def test_pornotube(self):
-        self._assert_restricted(
-            'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing',
-            '1689755.flv', 13)
-
 
 if __name__ == '__main__':
     unittest.main()

From 740a7fcbc8673a8edcfc9ebdab967ccb99fb8c08 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:11:17 +0100
Subject: [PATCH 456/974] [gdcvault] Skip test that is now restricted

---
 youtube_dl/extractor/gdcvault.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index d453ec010..fed968f51 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -39,7 +39,8 @@ class GDCVaultIE(InfoExtractor):
                 'id': '1015301',
                 'ext': 'flv',
                 'title': 'Thexder Meets Windows 95, or Writing Great Games in the Windows 95 Environment',
-            }
+            },
+            'skip': 'Requires login',
         }
     ]
 

From 9298d4e3df4b6237f4f9e4f37b9e55775c5e3c0c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:36:56 +0100
Subject: [PATCH 457/974] [discovery] Fix extractor

---
 youtube_dl/extractor/discovery.py | 46 +++++++++++++++----------------
 1 file changed, 22 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 52c2d7ddf..d3e667528 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -1,47 +1,45 @@
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    int_or_none,
+)
 
 
 class DiscoveryIE(InfoExtractor):
-    _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9\-]*)(.htm)?'
+    _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
     _TEST = {
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
-        'md5': 'e12614f9ee303a6ccef415cb0793eba2',
+        'md5': '3c69d77d9b0d82bfd5e5932a60f26504',
         'info_dict': {
-            'id': '614784',
-            'ext': 'mp4',
-            'title': 'MythBusters: Mission Impossible Outtakes',
+            'id': 'mission-impossible-outtakes',
+            'ext': 'flv',
+            'title': 'Mission Impossible Outtakes',
             'description': ('Watch Jamie Hyneman and Adam Savage practice being'
                             ' each other -- to the point of confusing Jamie\'s dog -- and '
                             'don\'t miss Adam moon-walking as Jamie ... behind Jamie\'s'
                             ' back.'),
             'duration': 156,
+            'timestamp': 1303099200,
+            'upload_date': '20110418',
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_list_json = self._search_regex(r'var videoListJSON = ({.*?});',
-                                             webpage, 'video list', flags=re.DOTALL)
-        video_list = json.loads(video_list_json)
-        info = video_list['clips'][0]
-        formats = []
-        for f in info['mp4']:
-            formats.append(
-                {'url': f['src'], 'ext': 'mp4', 'tbr': int(f['bitrate'][:-1])})
+        info = self._parse_json(self._search_regex(
+            r'(?s)<script type="application/ld\+json">(.*?)</script>',
+            webpage, 'video info'), video_id)
 
         return {
-            'id': info['contentId'],
-            'title': video_list['name'],
-            'formats': formats,
-            'description': info['videoCaption'],
-            'thumbnail': info.get('videoStillURL') or info.get('thumbnailURL'),
-            'duration': info['duration'],
+            'id': video_id,
+            'title': info['name'],
+            'url': info['contentURL'],
+            'description': info.get('description'),
+            'thumbnail': info.get('thumbnailUrl'),
+            'timestamp': parse_iso8601(info.get('uploadDate')),
+            'duration': int_or_none(info.get('duration')),
         }

From 92b065dc539649d29804ebe329c89de9503b7098 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:50:46 +0100
Subject: [PATCH 458/974] [tudou] Fix extraction

---
 youtube_dl/extractor/tudou.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 161e47624..c89de5ba4 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -9,7 +9,7 @@ from .common import InfoExtractor
 
 
 class TudouIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?tudou\.com/(?:listplay|programs|albumplay)/(?:view|(.+?))/(?:([^/]+)|([^/]+))(?:\.html)?'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/.*?/(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
         'md5': '140a49ed444bd22f93330985d8475fcb',
@@ -27,13 +27,6 @@ class TudouIE(InfoExtractor):
             'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }, {
-        'url': 'http://www.tudou.com/albumplay/TenTw_JgiPM/PzsAs5usU9A.html',
-        'info_dict': {
-            'title': 'todo.mp4',
-        },
-        'add_ie': ['Youku'],
-        'skip': 'Only works from China'
     }]
 
     def _url_for_id(self, id, quality=None):
@@ -45,8 +38,7 @@ class TudouIE(InfoExtractor):
         return final_url
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(2)
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         m = re.search(r'vcode:\s*[\'"](.+?)[\'"]', webpage)
@@ -87,4 +79,9 @@ class TudouIE(InfoExtractor):
             }
             result.append(part_info)
 
-        return result
+        return {
+            '_type': 'multi_video',
+            'entries': result,
+            'id': video_id,
+            'title': title,
+        }

From c0bdf32a3cb0f6a08ba92cb5261d020ef2f61892 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 18:01:41 +0100
Subject: [PATCH 459/974] Add --print-json (Closes #2845)

---
 youtube_dl/__init__.py | 9 +++++----
 youtube_dl/options.py  | 5 +++++
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 4aa7fba6a..844992d8e 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -197,7 +197,8 @@ def _real_main(argv=None):
                      ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
                      ' template'.format(outtmpl))
 
-    any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
+    any_getting = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
+    any_printing = opts.print_json
     download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
     # PostProcessors
@@ -243,7 +244,7 @@ def _real_main(argv=None):
         'password': opts.password,
         'twofactor': opts.twofactor,
         'videopassword': opts.videopassword,
-        'quiet': (opts.quiet or any_printing),
+        'quiet': (opts.quiet or any_getting or any_printing),
         'no_warnings': opts.no_warnings,
         'forceurl': opts.geturl,
         'forcetitle': opts.gettitle,
@@ -253,9 +254,9 @@ def _real_main(argv=None):
         'forceduration': opts.getduration,
         'forcefilename': opts.getfilename,
         'forceformat': opts.getformat,
-        'forcejson': opts.dumpjson,
+        'forcejson': opts.dumpjson or opts.print_json,
         'dump_single_json': opts.dump_single_json,
-        'simulate': opts.simulate or any_printing,
+        'simulate': opts.simulate or any_getting,
         'skip_download': opts.skip_download,
         'format': opts.format,
         'format_limit': opts.format_limit,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 058342dec..feaca03a3 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -445,6 +445,11 @@ def parseOpts(overrideArguments=None):
         '-J', '--dump-single-json',
         action='store_true', dest='dump_single_json', default=False,
         help='simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.')
+    verbosity.add_option(
+        '--print-json',
+        action='store_true', dest='print_json', default=False,
+        help='Be quiet and print the video information as JSON (video is still being downloaded).',
+    )
     verbosity.add_option(
         '--newline',
         action='store_true', dest='progress_with_newline', default=False,

From ff0813313ac33b6abd03e9322b706dc83c5aeb14 Mon Sep 17 00:00:00 2001
From: Your Name <you@example.com>
Date: Thu, 8 Jan 2015 18:35:33 -0600
Subject: [PATCH 460/974] Minor style changes

---
 youtube_dl/extractor/audiomack.py | 54 +++++++++++++++----------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index b6f3039c7..73cde78d3 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -9,21 +9,21 @@ import time
 
 
 class AudiomackIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/(song)/(?P<id>[\w/-]+)'
+    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P<id>[\w/-]+)'
     IE_NAME = 'audiomack'
     _TESTS = [
-        # audiomack
+        # hosted on audiomack
         {
             'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary',
             'info_dict':
             {
                 'id': '310086',
-                "ext": "mp3",
-                "artist": "Roosh Williams",
+                'ext': 'mp3',
+                'artist': 'Roosh Williams',
                 'title': 'Extraordinary'
             }
         },
-        # audiomack through soundcloud
+        # audiomack wrapper around soundcloud song
         {
             'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
@@ -41,12 +41,12 @@ class AudiomackIE(InfoExtractor):
     @staticmethod
     def create_song_dictionary(api_response, album_url_tag, track_no=0):
         # All keys are the same in audiomack api and InfoExtractor format
-        entry = {key: api_response[key] for key in ["title", "artist", "id", "url"] if key in api_response}
+        entry = {key: api_response[key] for key in ['title', 'artist', 'id', 'url'] if key in api_response}
         # Fudge values in the face of missing metadata
-        if "id" not in entry:
-            entry["id"] = track_no
-        if "title" not in entry:
-            entry["title"] = album_url_tag
+        if 'id' not in entry:
+            entry['id'] = track_no
+        if 'title' not in entry:
+            entry['title'] = album_url_tag
         return entry
 
     def _real_extract(self, url):
@@ -57,18 +57,18 @@ class AudiomackIE(InfoExtractor):
 
         # Request the extended version of the api for extra fields like artist and title
         api_response = self._download_json(
-            "http://www.audiomack.com/api/music/url/song/%s?extended=1&_=%d" % (
+            'http://www.audiomack.com/api/music/url/song/%s?extended=1&_=%d' % (
                 album_url_tag, time.time()),
             album_url_tag)
 
         # API is inconsistent with errors
-        if "url" not in api_response or not api_response["url"] or "error" in api_response:
-            raise ExtractorError("Invalid url %s", url)
+        if 'url' not in api_response or not api_response['url'] or 'error' in api_response:
+            raise ExtractorError('Invalid url %s', url)
 
         # Audiomack wraps a lot of soundcloud tracks in their branded wrapper
         # if so, pass the work off to the soundcloud extractor
-        if SoundcloudIE.suitable(api_response["url"]):
-            return {'_type': 'url', 'url': api_response["url"], 'ie_key': 'Soundcloud'}
+        if SoundcloudIE.suitable(api_response['url']):
+            return {'_type': 'url', 'url': api_response['url'], 'ie_key': 'Soundcloud'}
 
         return self.create_song_dictionary(api_response, album_url_tag)
 
@@ -80,17 +80,17 @@ class AudiomackAlbumIE(InfoExtractor):
         # Standard album playlist
         {
             'url': 'http://www.audiomack.com/album/flytunezcom/tha-tour-part-2-mixtape',
-            "playlist_count": 15,
+            'playlist_count': 15,
             'info_dict':
             {
-                'id': "812251",
-                'title': "Tha Tour: Part 2 (Official Mixtape)"
+                'id': '812251',
+                'title': 'Tha Tour: Part 2 (Official Mixtape)'
             }
         },
         # Album playlist ripped from fakeshoredrive with no metadata
         {
-            "url": "http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project",
-            "playlist_count": 10
+            'url': 'http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project',
+            'playlist_count': 10
         }
     ]
 
@@ -99,27 +99,27 @@ class AudiomackAlbumIE(InfoExtractor):
         # this title is whatever the user types in, and is rarely
         # the proper song title.  Real metadata is in the api response
         album_url_tag = self._match_id(url)
-        result = {"_type": "playlist", "entries": []}
+        result = {'_type': 'playlist', 'entries': []}
         # There is no one endpoint for album metadata - instead it is included/repeated in each song's metadata
         # Therefore we don't know how many songs the album has and must infi-loop until failure
         track_no = 0
         while True:
             # Get song's metadata
-            api_response = self._download_json("http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d"
+            api_response = self._download_json('http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d'
                                                % (album_url_tag, track_no, time.time()), album_url_tag)
 
             # Total failure, only occurs when url is totally wrong
             # Won't happen in middle of valid playlist (next case)
-            if "url" not in api_response or "error" in api_response:
-                raise ExtractorError("Invalid url for track %d of album url %s" % (track_no, url))
+            if 'url' not in api_response or 'error' in api_response:
+                raise ExtractorError('Invalid url for track %d of album url %s' % (track_no, url))
             # URL is good but song id doesn't exist - usually means end of playlist
-            elif not api_response["url"]:
+            elif not api_response['url']:
                 break
             else:
                 # Pull out the album metadata and add to result (if it exists)
-                for resultkey, apikey in [("id", "album_id"), ("title", "album_title")]:
+                for resultkey, apikey in [('id', 'album_id'), ('title', 'album_title')]:
                     if apikey in api_response and resultkey not in result:
                         result[resultkey] = api_response[apikey]
-                result["entries"].append(AudiomackIE.create_song_dictionary(api_response, album_url_tag, track_no))
+                result['entries'].append(AudiomackIE.create_song_dictionary(api_response, album_url_tag, track_no))
             track_no += 1
         return result

From b0e87c311078cf8510c720d50d4960a87f43296b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 19:02:06 +0100
Subject: [PATCH 461/974] [ffmpeg] Correctly encode paths on Windows

On Python 2.x on Windows, if there are any unicode arguments in the command argument list, the whole list is converted to unicode internally.
Therefore, we need to call encodeArgument on every argument.

Fixes #4337 and #4668.
---
 youtube_dl/postprocessor/ffmpeg.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 473536dcc..d1b342c7a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -80,8 +80,9 @@ class FFmpegPostProcessor(PostProcessor):
 
         files_cmd = []
         for path in input_paths:
-            files_cmd.extend(['-i', encodeFilename(path, True)])
-        cmd = ([self._executable, '-y'] + files_cmd
+            files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
+        cmd = ([encodeFilename(self._executable, True), encodeArgument('-y')] +
+               files_cmd
                + [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
 
@@ -122,8 +123,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
         try:
             cmd = [
-                self._probe_executable,
-                '-show_streams',
+                encodeFilename(self._probe_executable, True),
+                encodeArgument('-show_streams'),
                 encodeFilename(self._ffmpeg_filename_argument(path), True)]
             handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
             output = handle.communicate()[0]

From 5e4166478de0a8a7ddafd013919acd08b4211910 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 19:17:15 +0100
Subject: [PATCH 462/974] [README] Add an FAQ entry for how to install on
 Windows

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 24b4d7970..fe380619f 100644
--- a/README.md
+++ b/README.md
@@ -450,6 +450,14 @@ Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unz
 
 To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistributable Package](http://www.microsoft.com/en-us/download/details.aspx?id=29).
 
+### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files?
+
+If you put youtube-dl and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome.
+
+To make a different directory work - either for ffmpeg, or for youtube-dl, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory.
+
+From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.

From 9932a65370564e08be11b0611e52b63ff451742d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 9 Jan 2015 20:13:53 +0100
Subject: [PATCH 463/974] [vk] Remove debug assert statement (fixes #4672,
 fixes #4514)

---
 youtube_dl/extractor/vk.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 129de6cf3..81e02a624 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -166,7 +166,6 @@ class VKIE(InfoExtractor):
 
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
-        assert m_rutube
         if m_rutube is not None:
             self.to_screen('rutube video detected')
             rutube_url = self._proto_relative_url(

From aa2fd59857a3cc16e542dd93b762a607c98e9202 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 20:16:55 +0100
Subject: [PATCH 464/974] [update] Use utils HTTPS handler (Fixes #4666)

On FreeBSD, the default HTTPS handler is missing certificates, so use our own.
---
 youtube_dl/update.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 2d2703368..3f9c5249d 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -13,6 +13,7 @@ from .compat import (
     compat_str,
     compat_urllib_request,
 )
+from .utils import make_HTTPS_handler
 from .version import __version__
 
 
@@ -58,9 +59,12 @@ def update_self(to_screen, verbose):
         to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 
+    https_handler = make_HTTPS_handler(False)
+    opener = compat_urllib_request.build_opener(https_handler)
+
     # Check if there is a new version
     try:
-        newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip()
+        newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()
     except:
         if verbose:
             to_screen(compat_str(traceback.format_exc()))
@@ -72,7 +76,7 @@ def update_self(to_screen, verbose):
 
     # Download and check versions info
     try:
-        versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
+        versions_info = opener.open(JSON_URL).read().decode('utf-8')
         versions_info = json.loads(versions_info)
     except:
         if verbose:
@@ -120,7 +124,7 @@ def update_self(to_screen, verbose):
             return
 
         try:
-            urlh = compat_urllib_request.urlopen(version['exe'][0])
+            urlh = opener.open(version['exe'][0])
             newcontent = urlh.read()
             urlh.close()
         except (IOError, OSError):
@@ -166,7 +170,7 @@ start /b "" cmd /c del "%%~f0"&exit /b"
     # Zip unix package
     elif isinstance(globals().get('__loader__'), zipimporter):
         try:
-            urlh = compat_urllib_request.urlopen(version['bin'][0])
+            urlh = opener.open(version['bin'][0])
             newcontent = urlh.read()
             urlh.close()
         except (IOError, OSError):

From 754c838903778911174e2777aebf4d1348780198 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 20:20:55 +0100
Subject: [PATCH 465/974] release 2015.01.09

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index fe380619f..3d632a553 100644
--- a/README.md
+++ b/README.md
@@ -219,6 +219,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      for each command-line argument. If the URL
                                      refers to a playlist, dump the whole
                                      playlist information in a single line.
+    --print-json                     Be quiet and print the video information as
+                                     JSON (video is still being downloaded).
     --newline                        output progress bar as new lines
     --no-progress                    do not print progress bar
     --console-title                  display progress in console titlebar
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 32019e362..2d942852f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.08'
+__version__ = '2015.01.09'

From 230b2287dd75430d0dcc3c22b407d916cb608b77 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 20:44:20 +0100
Subject: [PATCH 466/974] [youtube] Add acodec information

The codec seems to be consistently aac, so state that in our metadata.
---
 youtube_dl/extractor/youtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e71956071..bc18276d6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -264,9 +264,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 128, 'preference': -50},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 256, 'preference': -50},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},

From f90ad27375206433cdcb6609816893cb3c29c70d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 20:50:23 +0100
Subject: [PATCH 467/974] [YoutubeDL] Copy over format metadata when merging
 (Fixes #4671)

---
 youtube_dl/YoutubeDL.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 24d6c2de7..e34b36016 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -913,6 +913,14 @@ class YoutubeDL(object):
                                 'requested_formats': formats_info,
                                 'format': rf,
                                 'ext': formats_info[0]['ext'],
+                                'width': formats_info[0].get('width'),
+                                'height': formats_info[0].get('height'),
+                                'resolution': formats_info[0].get('resolution'),
+                                'fps': formats_info[0].get('fps'),
+                                'vcodec': formats_info[0].get('vcodec'),
+                                'vbr': formats_info[0].get('vbr'),
+                                'acodec': formats_info[1].get('acodec'),
+                                'abr': formats_info[1].get('abr'),
                             }
                         else:
                             selected_format = None

From 7b6faddfc808ba798ab654312e98d01005cda9ee Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 20:52:49 +0100
Subject: [PATCH 468/974] [wdr] Modernize

---
 youtube_dl/extractor/wdr.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 8e25ecf28..d1c46ccb3 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -172,8 +172,7 @@ class WDRMausIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         param_code = self._html_search_regex(
@@ -224,5 +223,3 @@ class WDRMausIE(InfoExtractor):
             'thumbnail': thumbnail,
             'upload_date': upload_date,
         }
-
-# TODO test _1

From 176cf9e0c3ef9efd4864b940ac1c2a1c0236593a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 21:33:07 +0100
Subject: [PATCH 469/974] [wdr] Support overviews (Fixes #4651)

---
 youtube_dl/extractor/wdr.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index d1c46ccb3..45466e31b 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import itertools
 import re
 
 from .common import InfoExtractor
@@ -67,6 +68,10 @@ class WDRIE(InfoExtractor):
                 'upload_date': '20140717',
             },
         },
+        {
+            'url': 'http://www1.wdr.de/mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100.html',
+            'playlist_mincount': 146,
+        }
     ]
 
     def _real_extract(self, url):
@@ -81,6 +86,27 @@ class WDRIE(InfoExtractor):
                 self.url_result(page_url + href, 'WDR')
                 for href in re.findall(r'<a href="/?(.+?%s\.html)" rel="nofollow"' % self._PLAYER_REGEX, webpage)
             ]
+
+            if entries:  # Playlist page
+                return self.playlist_result(entries, page_id)
+
+            # Overview page
+            entries = []
+            for page_num in itertools.count(2):
+                hrefs = re.findall(
+                    r'<li class="mediathekvideo"\s*>\s*<img[^>]*>\s*<a href="(/mediathek/video/[^"]+)"',
+                    webpage)
+                entries.extend(
+                    self.url_result(page_url + href, 'WDR')
+                    for href in hrefs)
+                next_url_m = re.search(
+                    r'<li class="nextToLast">\s*<a href="([^"]+)"', webpage)
+                if not next_url_m:
+                    break
+                next_url = page_url + next_url_m.group(1)
+                webpage = self._download_webpage(
+                    next_url, page_id,
+                    note='Downloading playlist page %d' % page_num)
             return self.playlist_result(entries, page_id)
 
         flashvars = compat_parse_qs(

From 4baea47c42c291564dcbbbfc93955afbd4a80460 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 21:33:16 +0100
Subject: [PATCH 470/974] release 2015.01.09.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2d942852f..8270f7638 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.09'
+__version__ = '2015.01.09.1'

From b8da6b9fc6b05dff5dfbe98e20546b7e5e0ab170 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 22:43:49 +0100
Subject: [PATCH 471/974] [elpais] Modernize

---
 youtube_dl/extractor/elpais.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/elpais.py b/youtube_dl/extractor/elpais.py
index 4277202a2..00a69e631 100644
--- a/youtube_dl/extractor/elpais.py
+++ b/youtube_dl/extractor/elpais.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import unified_strdate
 
@@ -24,9 +22,7 @@ class ElPaisIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         prefix = self._html_search_regex(

From d120e9013f8dbab7e82fe1878e9b5e5e9bb94e88 Mon Sep 17 00:00:00 2001
From: Andrei Troie <andreitroie90@gmail.com>
Date: Fri, 9 Jan 2015 22:03:56 +0000
Subject: [PATCH 472/974] Added an option to specify an output format for
 merges when downloading separate video & audio

---
 youtube_dl/options.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index feaca03a3..0ed5e3db4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -302,6 +302,12 @@ def parseOpts(overrideArguments=None):
         '--youtube-skip-dash-manifest',
         action='store_false', dest='youtube_include_dash_manifest',
         help='Do not download the DASH manifest on YouTube videos')
+    video_format.add_option(
+    	'--merge-output-format',
+        action='store', dest='merge_output_format', metavar='FORMAT' ,default=None,
+        help=(
+            'If a merge is required (e.g. bestvideo+bestaudio), output to given container format (e.g. mkv, mp4, ogg, webm, flv) '
+            'Ignored if no merge is required'))
 
     subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
     subtitles.add_option(

From dd622d7c4ec15aec6ea5d66ed88a236147373e95 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 23:59:18 +0100
Subject: [PATCH 473/974] [netzkino] Add new extractor (Fixes #4669)

---
 test/helper.py                   | 14 ++++++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/common.py   | 11 ++++
 youtube_dl/extractor/netzkino.py | 86 ++++++++++++++++++++++++++++++++
 youtube_dl/utils.py              |  4 ++
 5 files changed, 116 insertions(+)
 create mode 100644 youtube_dl/extractor/netzkino.py

diff --git a/test/helper.py b/test/helper.py
index 77225e4f7..c416f388c 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -110,6 +110,20 @@ def expect_info_dict(self, got_dict, expected_dict):
         else:
             if isinstance(expected, compat_str) and expected.startswith('md5:'):
                 got = 'md5:' + md5(got_dict.get(info_field))
+            elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
+                got = got_dict.get(info_field)
+                self.assertTrue(
+                    isinstance(got, list),
+                    'Expected field %s to be a list, but it is of type %s' % (
+                        info_field, type(got).__name__))
+                expected_num = int(expected.partition(':')[2])
+                assertGreaterEqual(
+                    self, len(got), expected_num,
+                    'Expected %d items in field %s, but only got %d' % (
+                        expected_num, info_field, len(got)
+                    )
+                )
+                continue
             else:
                 got = got_dict.get(info_field)
             self.assertEqual(expected, got,
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8dacc2c54..5da7568ca 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,6 +274,7 @@ from .nbc import (
 )
 from .ndr import NDRIE
 from .ndtv import NDTVIE
+from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d703893dc..b4cd59e43 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -147,6 +147,17 @@ class InfoExtractor(object):
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video
     comment_count:  Number of comments on the video
+    comments:       A list of comments, each with one or more of the following
+                    properties (all but one of text or html optional):
+                        * "author" - human-readable name of the comment author
+                        * "author_id" - user ID of the comment author
+                        * "id" - Comment ID
+                        * "html" - Comment as HTML
+                        * "text" - Plain text of the comment
+                        * "timestamp" - UNIX timestamp of comment
+                        * "parent" - ID of the comment this one is replying to.
+                                     Set to "root" to indicate that this is a
+                                     comment to the original video.
     age_limit:      Age restriction for the video, as an integer (years)
     webpage_url:    The url to the video webpage, if given to youtube-dl it
                     should allow to get the same result again. (It will be set
diff --git a/youtube_dl/extractor/netzkino.py b/youtube_dl/extractor/netzkino.py
new file mode 100644
index 000000000..93567d1e3
--- /dev/null
+++ b/youtube_dl/extractor/netzkino.py
@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    int_or_none,
+    js_to_json,
+    parse_iso8601,
+)
+
+
+class NetzkinoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?netzkino\.de/\#!/(?P<category>[^/]+)/(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://www.netzkino.de/#!/scifikino/rakete-zum-mond',
+        'md5': '92a3f8b76f8d7220acce5377ea5d4873',
+        'info_dict': {
+            'id': 'rakete-zum-mond',
+            'ext': 'mp4',
+            'title': 'Rakete zum Mond (Endstation Mond, Destination Moon)',
+            'comments': 'mincount:3',
+            'description': 'md5:1eddeacc7e62d5a25a2d1a7290c64a28',
+            'upload_date': '20120813',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'timestamp': 1344858571,
+            'age_limit': 12,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        category_id = mobj.group('category')
+        video_id = mobj.group('id')
+
+        api_url = 'http://api.netzkino.de.simplecache.net/capi-2.0a/categories/%s.json?d=www' % category_id
+        api_info = self._download_json(api_url, video_id)
+        info = next(
+            p for p in api_info['posts'] if p['slug'] == video_id)
+        custom_fields = info['custom_fields']
+
+        production_js = self._download_webpage(
+            'http://www.netzkino.de/beta/dist/production.min.js', video_id,
+            note='Downloading player code')
+        avo_js = self._search_regex(
+            r'window\.avoCore\s*=.*?urlTemplate:\s*(\{.*?"\})',
+            production_js, 'URL templates')
+        templates = self._parse_json(
+            avo_js, video_id, transform_source=js_to_json)
+
+        suffix = {
+            'hds': '.mp4/manifest.f4m',
+            'hls': '.mp4/master.m3u8',
+            'pmd': '.mp4',
+        }
+        film_fn = custom_fields['Streaming'][0]
+        formats = [{
+            'format_id': key,
+            'ext': 'mp4',
+            'url': tpl.replace('{}', film_fn) + suffix[key],
+        } for key, tpl in templates.items()]
+        self._sort_formats(formats)
+
+        comments = [{
+            'timestamp': parse_iso8601(c.get('date'), delimiter=' '),
+            'id': c['id'],
+            'author': c['name'],
+            'html': c['content'],
+            'parent': 'root' if c.get('parent', 0) == 0 else c['parent'],
+        } for c in info.get('comments', [])]
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'comments': comments,
+            'title': info['title'],
+            'age_limit': int_or_none(custom_fields.get('FSK')[0]),
+            'timestamp': parse_iso8601(info.get('date'), delimiter=' '),
+            'description': clean_html(info.get('content')),
+            'thumbnail': info.get('thumbnail'),
+            'playlist_title': api_info.get('title'),
+            'playlist_id': category_id,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 29739a483..079e8d2c3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -205,6 +205,10 @@ def get_element_by_attribute(attribute, value, html):
 
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""
+
+    if html is None:  # Convenience for sanitizing descriptions etc.
+        return html
+
     # Newline vs <br />
     html = html.replace('\n', ' ')
     html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)

From 130239460370870c33f259ed1549878388996d3f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 23:59:29 +0100
Subject: [PATCH 474/974] release 2015.01.09.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8270f7638..8c57c7413 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.09.1'
+__version__ = '2015.01.09.2'

From 34c781a24d1cc9b424ddc3ab3c4c616f49642c6a Mon Sep 17 00:00:00 2001
From: Andrei Troie <andreitroie90@gmail.com>
Date: Sat, 10 Jan 2015 00:03:11 +0000
Subject: [PATCH 475/974] Passing the option into the main program's arguments

---
 youtube_dl/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 844992d8e..b4ae35a2b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -166,6 +166,9 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
             parser.error('invalid video recode format specified')
+    if opts.merge_output_format is not None and not '+' in opts.format: #if merge format output is used on videos that don't require merging, ignore
+        opts.merge_output_format = None
+        
     if opts.date is not None:
         date = DateRange.day(opts.date)
     else:
@@ -323,6 +326,7 @@ def _real_main(argv=None):
         'encoding': opts.encoding,
         'exec_cmd': opts.exec_cmd,
         'extract_flat': opts.extract_flat,
+        'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
     }
 

From d02115f83727ae0d4f394073235dc97086c64266 Mon Sep 17 00:00:00 2001
From: Andrei Troie <andreitroie90@gmail.com>
Date: Sat, 10 Jan 2015 00:29:06 +0000
Subject: [PATCH 476/974] Use the option in preparing the merge output filename

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 24d6c2de7..6d5c401df 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -912,7 +912,7 @@ class YoutubeDL(object):
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
-                                'ext': formats_info[0]['ext'],
+                                'ext': self.params['merge_output_format'] if self.params['merge_output_format'] is not None else formats_info[0]['ext'],
                             }
                         else:
                             selected_format = None

From bd1a281ede1548ae0f0c74360eb518751c2a4526 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 02:03:00 +0100
Subject: [PATCH 477/974] [options] PEP8 and simpler --merge-output-format
 handling (#4673)

---
 youtube_dl/__init__.py | 4 +---
 youtube_dl/options.py  | 6 +++---
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index b4ae35a2b..8e7b74466 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -166,9 +166,7 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
             parser.error('invalid video recode format specified')
-    if opts.merge_output_format is not None and not '+' in opts.format: #if merge format output is used on videos that don't require merging, ignore
-        opts.merge_output_format = None
-        
+
     if opts.date is not None:
         date = DateRange.day(opts.date)
     else:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0ed5e3db4..14006178d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -303,10 +303,10 @@ def parseOpts(overrideArguments=None):
         action='store_false', dest='youtube_include_dash_manifest',
         help='Do not download the DASH manifest on YouTube videos')
     video_format.add_option(
-    	'--merge-output-format',
-        action='store', dest='merge_output_format', metavar='FORMAT' ,default=None,
+        '--merge-output-format',
+        action='store', dest='merge_output_format', metavar='FORMAT', default=None,
         help=(
-            'If a merge is required (e.g. bestvideo+bestaudio), output to given container format (e.g. mkv, mp4, ogg, webm, flv) '
+            'If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.'
             'Ignored if no merge is required'))
 
     subtitles = optparse.OptionGroup(parser, 'Subtitle Options')

From 904fffffebcf26b8004d6298001f17236857a5d8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 02:24:46 +0100
Subject: [PATCH 478/974] [audiomack] Better titles, simplify code

---
 youtube_dl/extractor/audiomack.py | 60 +++++++++++++++++++------------
 1 file changed, 37 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 73cde78d3..8bfe50214 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -1,11 +1,15 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import itertools
+import time
+
 from .common import InfoExtractor
 from .soundcloud import SoundcloudIE
-from ..utils import ExtractorError
-
-import time
+from ..utils import (
+    ExtractorError,
+    url_basename,
+)
 
 
 class AudiomackIE(InfoExtractor):
@@ -19,7 +23,7 @@ class AudiomackIE(InfoExtractor):
             {
                 'id': '310086',
                 'ext': 'mp3',
-                'artist': 'Roosh Williams',
+                'uploader': 'Roosh Williams',
                 'title': 'Extraordinary'
             }
         },
@@ -38,17 +42,6 @@ class AudiomackIE(InfoExtractor):
         },
     ]
 
-    @staticmethod
-    def create_song_dictionary(api_response, album_url_tag, track_no=0):
-        # All keys are the same in audiomack api and InfoExtractor format
-        entry = {key: api_response[key] for key in ['title', 'artist', 'id', 'url'] if key in api_response}
-        # Fudge values in the face of missing metadata
-        if 'id' not in entry:
-            entry['id'] = track_no
-        if 'title' not in entry:
-            entry['title'] = album_url_tag
-        return entry
-
     def _real_extract(self, url):
         # URLs end with [uploader name]/[uploader title]
         # this title is whatever the user types in, and is rarely
@@ -70,7 +63,12 @@ class AudiomackIE(InfoExtractor):
         if SoundcloudIE.suitable(api_response['url']):
             return {'_type': 'url', 'url': api_response['url'], 'ie_key': 'Soundcloud'}
 
-        return self.create_song_dictionary(api_response, album_url_tag)
+        return {
+            'id': api_response.get('id', album_url_tag),
+            'uploader': api_response.get('artist'),
+            'title': api_response.get('title'),
+            'url': api_response['url'],
+        }
 
 
 class AudiomackAlbumIE(InfoExtractor):
@@ -90,7 +88,17 @@ class AudiomackAlbumIE(InfoExtractor):
         # Album playlist ripped from fakeshoredrive with no metadata
         {
             'url': 'http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project',
-            'playlist_count': 10
+            'playlist': [{
+                'info_dict': {
+                    'title': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
+                    'id': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
+                    'ext': 'mp3',
+                }
+            }],
+            'params': {
+                'playliststart': 8,
+                'playlistend': 8,
+            }
         }
     ]
 
@@ -102,11 +110,12 @@ class AudiomackAlbumIE(InfoExtractor):
         result = {'_type': 'playlist', 'entries': []}
         # There is no one endpoint for album metadata - instead it is included/repeated in each song's metadata
         # Therefore we don't know how many songs the album has and must infi-loop until failure
-        track_no = 0
-        while True:
+        for track_no in itertools.count():
             # Get song's metadata
-            api_response = self._download_json('http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d'
-                                               % (album_url_tag, track_no, time.time()), album_url_tag)
+            api_response = self._download_json(
+                'http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d'
+                % (album_url_tag, track_no, time.time()), album_url_tag,
+                note='Querying song information (%d)' % (track_no + 1))
 
             # Total failure, only occurs when url is totally wrong
             # Won't happen in middle of valid playlist (next case)
@@ -120,6 +129,11 @@ class AudiomackAlbumIE(InfoExtractor):
                 for resultkey, apikey in [('id', 'album_id'), ('title', 'album_title')]:
                     if apikey in api_response and resultkey not in result:
                         result[resultkey] = api_response[apikey]
-                result['entries'].append(AudiomackIE.create_song_dictionary(api_response, album_url_tag, track_no))
-            track_no += 1
+                song_id = url_basename(api_response['url']).rpartition('.')[0]
+                result['entries'].append({
+                    'id': api_response.get('id', song_id),
+                    'uploader': api_response.get('artist'),
+                    'title': api_response.get('title', song_id),
+                    'url': api_response['url'],
+                })
         return result

From fb4b030aafee72801b47d4715391bd5822f80e13 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 02:38:27 +0100
Subject: [PATCH 479/974] [tvp] Update tests and improve output

---
 youtube_dl/extractor/tvp.py | 164 +++++++++++++++++-------------------
 1 file changed, 78 insertions(+), 86 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index cf21d1202..cc26f417a 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -8,79 +8,62 @@ from .common import InfoExtractor
 
 class TvpIE(InfoExtractor):
     IE_NAME = 'tvp.pl'
-    _VALID_URL = r'https?://(?P<type>vod|www)\.tvp\.pl/.*/(?P<id>\d+)$'
+    _VALID_URL = r'https?://(?:vod|www)\.tvp\.pl/.*/(?P<id>\d+)$'
 
-    _TESTS = [
-        {
-            'url': 'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
-            'info_dict': {
-                'id': '12878238',
-                'ext': 'wmv',
-                'title': 'CAMPUSnews, 31.10.2013 - Odcinek 2',
-                'description': '',
-            },
-            'skip': 'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.',
-        }, {
-            'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem/wideo/odc-2/4278035',
-            'info_dict': {
-                'id': '4278035',
-                'ext': 'wmv',
-                'title': 'Ogniem i mieczem, odc. 2',
-                'description': 'Bohun dowiaduje się o złamaniu przez kniahinię danego mu słowa i wyrusza do Rozłogów. Helenie w ostatniej chwili udaje się uciec dzięki pomocy Zagłoby.',
-            },
-            'skip': 'As above',
-        }, {
-            'url': 'http://vod.tvp.pl/seriale/obyczajowe/czas-honoru/sezon-1-1-13/i-seria-odc-13/194536',
-            'info_dict': {
-                'id': '194536',
-                'ext': 'mp4',
-                'title': 'Czas honoru, I seria – odc. 13',
-                'description': 'WŁADEK\nCzesław prosi Marię o dostarczenie Władkowi zarazki tyfusu. Jeśli zachoruje zostanie przewieziony do szpitala skąd łatwiej będzie go odbić. Czy matka zdecyduje się zarazić syna? Karol odwiedza Wandę przyznaje się, że ją oszukiwał, ale ostrzega też, że grozi jej aresztowanie i nalega, żeby wyjechała z Warszawy. Czy dziewczyna zdecyduje się znów oddalić od ukochanego? Rozpoczyna się akcja odbicia Władka.',
-            },
-        }, {
-            'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
-            'info_dict': {
-                'id': '17916176',
-                'ext': 'mp4',
-                'title': 'rozmaitosci, TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
-                'description': '',
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': 'true',
-            },
-        }, {
-            'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
-            'info_dict': {
-                'id': '17834272',
-                'ext': 'mp4',
-                'title': 'Na sygnale, odc. 39',
-                'description': 'Ekipa Wiktora ratuje młodą matkę, która spadła ze schodów trzymając na rękach noworodka. Okazuje się, że dziewczyna jest surogatką, a biologiczni rodzice dziecka próbują zmusić ją do oddania synka…',
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': 'true',
-            },
+    _TESTS = [{
+        'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem/wideo/odc-2/4278035',
+        'info_dict': {
+            'id': '4278035',
+            'ext': 'wmv',
+            'title': 'Ogniem i mieczem, odc. 2',
+            'description': 'Bohun dowiaduje się o złamaniu przez kniahinię danego mu słowa i wyrusza do Rozłogów. Helenie w ostatniej chwili udaje się uciec dzięki pomocy Zagłoby.',
         },
-    ]
+    }, {
+        'url': 'http://vod.tvp.pl/seriale/obyczajowe/czas-honoru/sezon-1-1-13/i-seria-odc-13/194536',
+        'info_dict': {
+            'id': '194536',
+            'ext': 'mp4',
+            'title': 'Czas honoru, I seria – odc. 13',
+            #  'description': 'WŁADEK\nCzesław prosi Marię o dostarczenie Władkowi zarazki tyfusu. Jeśli zachoruje zostanie przewieziony do szpitala skąd łatwiej będzie go odbić. Czy matka zdecyduje się zarazić syna? Karol odwiedza Wandę przyznaje się, że ją oszukiwał, ale ostrzega też, że grozi jej aresztowanie i nalega, żeby wyjechała z Warszawy. Czy dziewczyna zdecyduje się znów oddalić od ukochanego? Rozpoczyna się akcja odbicia Władka.',
+        },
+    }, {
+        'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
+        'info_dict': {
+            'id': '17916176',
+            'ext': 'mp4',
+            'title': 'TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': 'true',
+        },
+    }, {
+        'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
+        'info_dict': {
+            'id': '17834272',
+            'ext': 'mp4',
+            'title': 'Na sygnale, odc. 39',
+            'description': 'Ekipa Wiktora ratuje młodą matkę, która spadła ze schodów trzymając na rękach noworodka. Okazuje się, że dziewczyna jest surogatką, a biologiczni rodzice dziecka próbują zmusić ją do oddania synka…',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': 'true',
+        },
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(
             'http://www.tvp.pl/sess/tvplayer.php?object_id=%s' % video_id, video_id)
+
         title = self._og_search_title(webpage)
         series = self._search_regex(
             r'{name:\s*([\'"])SeriesTitle\1,\s*value:\s*\1(?P<series>.*?)\1},',
             webpage, 'series', group='series', default=None)
         if series is not None and series not in title:
             title = '%s, %s' % (series, title)
-        info_dict = {
-            'id': video_id,
-            'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': self._og_search_description(webpage),
-        }
+        description = self._og_search_description(webpage, default=None)
+
         video_url = self._search_regex(
             r'0:{src:([\'"])(?P<url>.*?)\1', webpage, 'formats', group='url', default=None)
         if video_url is None:
@@ -92,55 +75,64 @@ class TvpIE(InfoExtractor):
         if ext != 'ism/manifest':
             if '/' in ext:
                 ext = 'mp4'
-            info_dict.update({
-                'ext': ext,
+            formats = [{
+                'format_id': 'direct',
                 'url': video_url,
-            })
+                'ext': ext,
+            }]
         else:
             m3u8_url = re.sub('([^/]*)\.ism/manifest', r'\1.ism/\1.m3u8', video_url)
             formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
-            info_dict.update({
-                'formats': formats,
-            })
-        return info_dict
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': description,
+            'formats': formats,
+        }
 
 
 class TvpSeriesIE(InfoExtractor):
     IE_NAME = 'tvp.pl:Series'
     _VALID_URL = r'https?://vod\.tvp\.pl/(?:[^/]+/){2}(?P<id>[^/]+)/?$'
 
-    _TESTS = [
-        {
-            'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem',
-            'info_dict': {
-                'title': 'Ogniem i mieczem',
-                'id': '4278026',
-            },
-            'playlist_count': 4,
-        }, {
-            'url': 'http://vod.tvp.pl/audycje/podroze/boso-przez-swiat',
-            'info_dict': {
-                'title': 'Boso przez świat',
-                'id': '9329207',
-            },
-            'playlist_count': 86,
-        }
-    ]
+    _TESTS = [{
+        'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem',
+        'info_dict': {
+            'title': 'Ogniem i mieczem',
+            'id': '4278026',
+        },
+        'playlist_count': 4,
+    }, {
+        'url': 'http://vod.tvp.pl/audycje/podroze/boso-przez-swiat',
+        'info_dict': {
+            'title': 'Boso przez świat',
+            'id': '9329207',
+        },
+        'playlist_count': 86,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id, tries=5)
+
         title = self._html_search_regex(
             r'(?s) id=[\'"]path[\'"]>(?:.*? / ){2}(.*?)</span>', webpage, 'series')
         playlist_id = self._search_regex(r'nodeId:\s*(\d+)', webpage, 'playlist id')
         playlist = self._download_webpage(
             'http://vod.tvp.pl/vod/seriesAjax?type=series&nodeId=%s&recommend'
-            'edId=0&sort=&page=0&pageSize=10000' % playlist_id, display_id, tries=5)
+            'edId=0&sort=&page=0&pageSize=10000' % playlist_id, display_id, tries=5,
+            note='Downloading playlist')
+
         videos_paths = re.findall(
             '(?s)class="shortTitle">.*?href="(/[^"]+)', playlist)
         entries = [
             self.url_result('http://vod.tvp.pl%s' % v_path, ie=TvpIE.ie_key())
             for v_path in videos_paths]
+
         return {
             '_type': 'playlist',
             'id': playlist_id,

From 6271f1cad9331cbdb274906d8330736fb888bda2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 05:45:51 +0100
Subject: [PATCH 480/974] [youtube|ffmpeg] Automatically correct video with
 non-square pixels (Fixes #4674)

---
 youtube_dl/YoutubeDL.py              | 29 ++++++++++++++++++++++++++++
 youtube_dl/__init__.py               |  1 +
 youtube_dl/extractor/common.py       |  3 +++
 youtube_dl/extractor/youtube.py      | 24 +++++++++++++++++++++++
 youtube_dl/options.py                |  7 +++++++
 youtube_dl/postprocessor/__init__.py |  2 ++
 youtube_dl/postprocessor/ffmpeg.py   | 23 ++++++++++++++++++++++
 7 files changed, 89 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 61675d8ec..4cc3ec2fb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -70,6 +70,7 @@ from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
+    FFmpegFixupStretchedPP,
     FFmpegMergerPP,
     FFmpegPostProcessor,
     get_postprocessor,
@@ -204,6 +205,12 @@ class YoutubeDL(object):
                        Progress hooks are guaranteed to be called at least once
                        (with status "finished") if the download is successful.
     merge_output_format: Extension to use when merging formats.
+    fixup:             Automatically correct known faults of the file.
+                       One of:
+                       - "never": do nothing
+                       - "warn": only emit a warning
+                       - "detect_or_warn": check whether we can do anything
+                                           about it, warn otherwise
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -924,6 +931,7 @@ class YoutubeDL(object):
                                 'fps': formats_info[0].get('fps'),
                                 'vcodec': formats_info[0].get('vcodec'),
                                 'vbr': formats_info[0].get('vbr'),
+                                'stretched_ratio': formats_info[0].get('stretched_ratio'),
                                 'acodec': formats_info[1].get('acodec'),
                                 'abr': formats_info[1].get('abr'),
                                 'ext': output_ext,
@@ -1154,6 +1162,27 @@ class YoutubeDL(object):
                     return
 
             if success:
+                # Fixup content
+                stretched_ratio = info_dict.get('stretched_ratio')
+                if stretched_ratio is not None and stretched_ratio != 1:
+                    fixup_policy = self.params.get('fixup')
+                    if fixup_policy is None:
+                        fixup_policy = 'detect_or_warn'
+                    if fixup_policy == 'warn':
+                        self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
+                            info_dict['id'], stretched_ratio))
+                    elif fixup_policy == 'detect_or_warn':
+                        stretched_pp = FFmpegFixupStretchedPP(self)
+                        if stretched_pp.available:
+                            info_dict.setdefault('__postprocessors', [])
+                            info_dict['__postprocessors'].append(stretched_pp)
+                        else:
+                            self.report_warning(
+                                '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
+                                    info_dict['id'], stretched_ratio))
+                    else:
+                        assert fixup_policy == 'ignore'
+
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 8e7b74466..659a92a3b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -326,6 +326,7 @@ def _real_main(argv=None):
         'extract_flat': opts.extract_flat,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
+        'fixup': opts.fixup,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cd155a090..363e2000c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -114,6 +114,9 @@ class InfoExtractor(object):
                                  to add to the request.
                     * http_post_data  Additional data to send with a POST
                                  request.
+                    * stretched_ratio  If given and not 1, indicates that the
+                                       video's pixels are not square.
+                                       width : height ratio as float.
     url:            Final video URL.
     ext:            Video filename extension.
     format:         The video format, defaults to ext (used for --get-format)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bc18276d6..c7611a3a0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -465,6 +465,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'skip_download': 'requires avconv',
             }
         },
+        # Non-square pixels
+        {
+            'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
+            'info_dict': {
+                'id': '_b-2C3KPAM0',
+                'ext': 'mp4',
+                'stretched_ratio': 16 / 9.,
+                'upload_date': '20110310',
+                'uploader_id': 'AllenMeow',
+                'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
+                'uploader': '孫艾倫',
+                'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
+            },
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -1051,6 +1065,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                             f['preference'] = f.get('preference', 0) - 10000
                     formats.extend(dash_formats)
 
+        # Check for malformed aspect ratio
+        stretched_m = re.search(
+            r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
+            video_webpage)
+        if stretched_m:
+            ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
+            for f in formats:
+                if f.get('vcodec') != 'none':
+                    f['stretched_ratio'] = ratio
+
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 14006178d..e5602bb3a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -631,6 +631,13 @@ def parseOpts(overrideArguments=None):
         '--xattrs',
         action='store_true', dest='xattrs', default=False,
         help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
+    postproc.add_option(
+        '--fixup',
+        metavar='POLICY', dest='fixup', default='detect_or_warn',
+        help='(experimental) Automatically correct known faults of the file. '
+             'One of never (do nothing), warn (only emit a warning), '
+             'detect_or_warn(check whether we can do anything about it, warn '
+             'otherwise')
     postproc.add_option(
         '--prefer-avconv',
         action='store_false', dest='prefer_ffmpeg',
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 7f505b58e..f8507951c 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -6,6 +6,7 @@ from .ffmpeg import (
     FFmpegAudioFixPP,
     FFmpegEmbedSubtitlePP,
     FFmpegExtractAudioPP,
+    FFmpegFixupStretchedPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
     FFmpegVideoConvertorPP,
@@ -24,6 +25,7 @@ __all__ = [
     'FFmpegAudioFixPP',
     'FFmpegEmbedSubtitlePP',
     'FFmpegExtractAudioPP',
+    'FFmpegFixupStretchedPP',
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
     'FFmpegPostProcessor',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index d1b342c7a..6e9194fa6 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -50,6 +50,10 @@ class FFmpegPostProcessor(PostProcessor):
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
         return dict((p, get_exe_version(p, args=['-version'])) for p in programs)
 
+    @property
+    def available(self):
+        return self._executable is not None
+
     @property
     def _executable(self):
         if self._downloader.params.get('prefer_ffmpeg', False):
@@ -540,3 +544,22 @@ class FFmpegAudioFixPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         return True, info
+
+
+class FFmpegFixupStretchedPP(FFmpegPostProcessor):
+    def run(self, info):
+        stretched_ratio = info.get('stretched_ratio')
+        if stretched_ratio is None or stretched_ratio == 1:
+            return
+
+        filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
+
+        options = ['-c', 'copy', '-aspect', '%f' % stretched_ratio]
+        self._downloader.to_screen('[ffmpeg] Fixing aspect ratio in "%s"' % filename)
+        self.run_ffmpeg(filename, temp_filename, options)
+
+        os.remove(encodeFilename(filename))
+        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+
+        return True, info

From f71fdb0acc0e168d2a5e4f60be96217647cc872e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 05:51:07 +0100
Subject: [PATCH 481/974] [eighttracks] Improve waiting (#3954)

---
 youtube_dl/extractor/eighttracks.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index 9df2141c3..a9b18195b 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -4,11 +4,12 @@ from __future__ import unicode_literals
 import json
 import random
 import re
-import time
 
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
+)
+from ..utils import (
     ExtractorError,
 )
 
@@ -115,7 +116,7 @@ class EightTracksIE(InfoExtractor):
         mix_id = data['id']
         track_count = data['tracks_count']
         duration = data['duration']
-        avg_song_duration = duration / track_count
+        avg_song_duration = float(duration) / track_count
         first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
         next_url = first_url
         entries = []
@@ -136,7 +137,7 @@ class EightTracksIE(InfoExtractor):
                         raise
                     else:
                         download_tries += 1
-                        time.sleep(avg_song_duration)
+                        self._sleep(avg_song_duration, playlist_id)
 
             api_data = json.loads(api_json)
             track_data = api_data['set']['track']

From 6ef9f88299e1595bbe467fb527ada9b003ae5779 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 05:51:22 +0100
Subject: [PATCH 482/974] release 2015.01.10

---
 README.md             | 9 +++++++++
 youtube_dl/version.py | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3d632a553..87aaec8a3 100644
--- a/README.md
+++ b/README.md
@@ -272,6 +272,10 @@ which means you can modify it, redistribute it or use it however you like.
     -F, --list-formats               list all available formats
     --youtube-skip-dash-manifest     Do not download the DASH manifest on
                                      YouTube videos
+    --merge-output-format FORMAT     If a merge is required (e.g.
+                                     bestvideo+bestaudio), output to given
+                                     container format. One of mkv, mp4, ogg,
+                                     webm, flv.Ignored if no merge is required
 
 ## Subtitle Options:
     --write-sub                      write subtitle file
@@ -318,6 +322,11 @@ which means you can modify it, redistribute it or use it however you like.
     --add-metadata                   write metadata to the video file
     --xattrs                         write metadata to the video file's xattrs
                                      (using dublin core and xdg standards)
+    --fixup POLICY                   (experimental) Automatically correct known
+                                     faults of the file. One of never (do
+                                     nothing), warn (only emit a warning),
+                                     detect_or_warn(check whether we can do
+                                     anything about it, warn otherwise
     --prefer-avconv                  Prefer avconv over ffmpeg for running the
                                      postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8c57c7413..60258d36a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.09.2'
+__version__ = '2015.01.10'

From 52afb2ac1b6551350cf4d28ad0d758a136186297 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 06:13:18 +0100
Subject: [PATCH 483/974] [ffmpeg] Call encodeFilename on filenames

---
 youtube_dl/postprocessor/ffmpeg.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 8bf5bebc3..26b99e43c 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -82,7 +82,8 @@ class FFmpegPostProcessor(PostProcessor):
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
         self.check_version()
 
-        oldest_mtime = min(os.stat(path).st_mtime for path in input_paths)
+        oldest_mtime = min(
+            os.stat(encodeFilename(path)).st_mtime for path in input_paths)
 
         files_cmd = []
         for path in input_paths:
@@ -100,7 +101,7 @@ class FFmpegPostProcessor(PostProcessor):
             stderr = stderr.decode('utf-8', 'replace')
             msg = stderr.strip().split('\n')[-1]
             raise FFmpegPostProcessorError(msg)
-        os.utime(out_path, (oldest_mtime, oldest_mtime))
+        os.utime(encodeFilename(out_path), (oldest_mtime, oldest_mtime))
         if self._deletetempfiles:
             for ipath in input_paths:
                 os.remove(ipath)

From b9f030cc26891fa8ee680742f5db17f819e5e81a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 14:23:54 +0100
Subject: [PATCH 484/974] [orf] Fix typo

---
 youtube_dl/extractor/orf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 4fed83bd6..3d2e5d2c9 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -160,7 +160,7 @@ class ORFOE1IE(InfoExtractor):
 
 
 class ORFFM4IE(InfoExtractor):
-    IE_DESC = 'orf:fm4'
+    IE_NAME = 'orf:fm4'
     IE_DESC = 'radio FM4'
     _VALID_URL = r'http://fm4\.orf\.at/7tage/?#(?P<date>[0-9]+)/(?P<show>\w+)'
 

From 385f8ae4684e7c7fd04d66b6066d980c93e2d79f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 14:25:11 +0100
Subject: [PATCH 485/974] [eighttracks] PEP8

---
 youtube_dl/extractor/eighttracks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index a9b18195b..fb5dbbe2b 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -122,7 +122,6 @@ class EightTracksIE(InfoExtractor):
         entries = []
 
         for i in range(track_count):
-            
             api_json = None
             download_tries = 0
 

From 27ca82ebc66c91359173e45b25d992118511e330 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 14:27:27 +0100
Subject: [PATCH 486/974] [orf:oe1] Add konsole URL schema (Fixes #4675)

---
 youtube_dl/extractor/orf.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 3d2e5d2c9..4e293392b 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -128,13 +128,16 @@ class ORFTVthekIE(InfoExtractor):
         }
 
 
-# Audios on ORF radio are only available for 7 days, so we can't add tests.
-
-
 class ORFOE1IE(InfoExtractor):
     IE_NAME = 'orf:oe1'
     IE_DESC = 'Radio Österreich 1'
-    _VALID_URL = r'http://oe1\.orf\.at/programm/(?P<id>[0-9]+)'
+    _VALID_URL = r'http://oe1\.orf\.at/(?:programm/|konsole.*?#\?track_id=)(?P<id>[0-9]+)'
+
+    # Audios on ORF radio are only available for 7 days, so we can't add tests.
+    _TEST = {
+        'url': 'http://oe1.orf.at/konsole?show=on_demand#?track_id=394211',
+        'only_matching': True,
+    }
 
     def _real_extract(self, url):
         show_id = self._match_id(url)

From 776dc3992a8995d22c172f7616d4ffd9cabc884e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Jan 2015 19:43:52 +0600
Subject: [PATCH 487/974] [utils] Clarify more day-month-first ambiguous
 formats

---
 youtube_dl/utils.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 079e8d2c3..a12b0a7de 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -679,9 +679,6 @@ def unified_strdate(date_str, day_first=True):
         '%b %dth %Y %I:%M%p',
         '%Y-%m-%d',
         '%Y/%m/%d',
-        '%d.%m.%Y',
-        '%d/%m/%Y',
-        '%d/%m/%y',
         '%Y/%m/%d %H:%M:%S',
         '%Y-%m-%d %H:%M:%S',
         '%Y-%m-%d %H:%M:%S.%f',
@@ -696,10 +693,16 @@ def unified_strdate(date_str, day_first=True):
     ]
     if day_first:
         format_expressions.extend([
+            '%d.%m.%Y',
+            '%d/%m/%Y',
+            '%d/%m/%y',
             '%d/%m/%Y %H:%M:%S',
         ])
     else:
         format_expressions.extend([
+            '%m.%d.%Y',
+            '%m/%d/%Y',
+            '%m/%d/%y',
             '%m/%d/%Y %H:%M:%S',
         ])
     for expression in format_expressions:

From d89c6e336a3a20085b4ea35f2afcfcecedab5c53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Jan 2015 19:44:29 +0600
Subject: [PATCH 488/974] [atttechchannel] Add extractor (Closes #3938)

---
 youtube_dl/extractor/__init__.py       |  1 +
 youtube_dl/extractor/atttechchannel.py | 55 ++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/atttechchannel.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a8579d083..b908eb38c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -26,6 +26,7 @@ from .arte import (
     ArteTVEmbedIE,
 )
 from .atresplayer import AtresPlayerIE
+from .atttechchannel import ATTTechChannelIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .auengine import AUEngineIE
 from .azubu import AzubuIE
diff --git a/youtube_dl/extractor/atttechchannel.py b/youtube_dl/extractor/atttechchannel.py
new file mode 100644
index 000000000..b01d35bb2
--- /dev/null
+++ b/youtube_dl/extractor/atttechchannel.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import unified_strdate
+
+
+class ATTTechChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://techchannel\.att\.com/play-video\.cfm/([^/]+/)*(?P<id>.+)'
+    _TEST = {
+        'url': 'http://techchannel.att.com/play-video.cfm/2014/1/27/ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
+        'info_dict': {
+            'id': '11316',
+            'display_id': 'ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
+            'ext': 'flv',
+            'title': 'AT&T Archives : The UNIX System: Making Computers Easier to Use',
+            'description': 'A 1982 film about UNIX is the foundation for software in use around Bell Labs and AT&T.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20140127',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_url = self._search_regex(
+            r"url\s*:\s*'(rtmp://[^']+)'",
+            webpage, 'video URL')
+
+        video_id = self._search_regex(
+            r'mediaid\s*=\s*(\d+)',
+            webpage, 'video id', fatal=False)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        upload_date = unified_strdate(self._search_regex(
+            r'[Rr]elease\s+date:\s*(\d{1,2}/\d{1,2}/\d{4})',
+            webpage, 'upload date', fatal=False), False)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'ext': 'flv',
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+        }

From 76bfaf6daf73ac74f7a9068c51d14ab0a5ec5e52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 17:46:01 +0100
Subject: [PATCH 489/974] [nrk] Improve subtitle support (#3092)

---
 youtube_dl/extractor/nrk.py | 42 ++++++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 321ce5ce7..f6de26022 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -7,8 +7,10 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     float_or_none,
+    parse_duration,
     unified_strdate,
 )
+from .subtitles import SubtitlesInfoExtractor
 
 
 class NRKIE(InfoExtractor):
@@ -71,8 +73,8 @@ class NRKIE(InfoExtractor):
         }
 
 
-class NRKTVIE(InfoExtractor):
-    _VALID_URL = r'http://tv\.nrk(?:super)?\.no/(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
+class NRKTVIE(SubtitlesInfoExtractor):
+    _VALID_URL = r'(?P<baseurl>http://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
         {
@@ -147,6 +149,29 @@ class NRKTVIE(InfoExtractor):
         }
     ]
 
+    def _seconds2str(self, s):
+        return '%02d:%02d:%02d.%03d' % (s / 3600, (s % 3600) / 60, s % 60, (s % 1) * 1000)
+
+    def _debug_print(self, txt):
+        if self._downloader.params.get('verbose', False):
+            self.to_screen('[debug] %s' % txt)
+
+    def _extract_captions(self, subtitlesurl, video_id, baseurl):
+        url = "%s%s" % (baseurl, subtitlesurl)
+        self._debug_print('%s: Subtitle url: %s' % (video_id, url))
+        captions = self._download_xml(url, video_id, 'Downloading subtitles')
+        lang = captions.get('lang', 'no')
+        ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/ns/ttml}'))
+        srt = ''
+        for pos, p in enumerate(ps):
+            begin = parse_duration(p.get('begin'))
+            duration = parse_duration(p.get('dur'))
+            starttime = self._seconds2str(begin)
+            endtime = self._seconds2str(begin + duration)
+            text = '\n'.join(p.itertext())
+            srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), starttime, endtime, text)
+        return {lang: srt}
+
     def _extract_f4m(self, manifest_url, video_id):
         return self._extract_f4m_formats(manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id)
 
@@ -154,6 +179,7 @@ class NRKTVIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         part_id = mobj.group('part_id')
+        baseurl = mobj.group('baseurl')
 
         webpage = self._download_webpage(url, video_id)
 
@@ -210,9 +236,18 @@ class NRKTVIE(InfoExtractor):
         m3u8_url = re.search(r'data-hls-media="([^"]+)"', webpage)
         if m3u8_url:
             formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4'))
-
         self._sort_formats(formats)
 
+        subtitles_url = self._html_search_regex(
+            r'data-subtitlesurl[ ]*=[ ]*"([^"]+)"',
+            webpage, 'subtitle URL', default=None)
+        subtitles = None
+        if subtitles_url:
+            subtitles = self._extract_captions(subtitles_url, video_id, baseurl)
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
         return {
             'id': video_id,
             'title': title,
@@ -221,4 +256,5 @@ class NRKTVIE(InfoExtractor):
             'upload_date': upload_date,
             'duration': duration,
             'formats': formats,
+            'subtitles': subtitles,
         }

From ea1d5bdcddc4b9bdac0ef5555f080484c33467a9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 17:57:21 +0100
Subject: [PATCH 490/974] [rte] Make more robust and add a new testcase (#4015)

---
 youtube_dl/extractor/rte.py | 38 ++++++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 8d4e94e55..b5a67373b 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -9,34 +9,38 @@ from ..utils import (
 
 
 class RteIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/in/show/(?P<id>[0-9]+)/'
+    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/(?P<id>[0-9]+)/'
     _TEST = {
-        'url': 'http://www.rte.ie/player/in/show/10336191/',
+        'url': 'http://www.rte.ie/player/de/show/10363114/',
         'info_dict': {
-            'id': '10336191',
+            'id': '10363114',
             'ext': 'mp4',
-            'title': 'Nine News',
+            'title': 'One News',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'The One O\'Clock News followed by Weather.',
-            'duration': 1622.963,
+            'duration': 436.844,
+        },
+        'params': {
+            'skip_download': 'f4m fails with --test atm'
         }
     }
     
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
+
         title = self._og_search_title(webpage)
-        
-        description = self._search_regex(r'<meta name="description" content="(.*?)" />', webpage, 'description')
-        duration = float_or_none(self._html_search_meta('duration', webpage, 'duration'), 1000)
-        
-        thumbnail_id = self._search_regex(r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
-        thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg' 
-        
+        description = self._html_search_meta('description', webpage, 'description')
+        duration = float_or_none(self._html_search_meta(
+            'duration', webpage, 'duration', fatal=False), 1000)
+
+        thumbnail_id = self._search_regex(
+            r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
+        thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg'
+
         feeds_url = self._html_search_meta("feeds-prefix", webpage, 'feeds url') + video_id
         json_string = self._download_json(feeds_url, video_id)
-        
+
         # f4m_url = server + relative_url
         f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url']
         f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
@@ -46,8 +50,8 @@ class RteIE(InfoExtractor):
             'ext': 'mp4',
             'width': f['width'],
             'height': f['height'],
-        } for f in f4m_formats ]
-        
+        } for f in f4m_formats]
+
         return {
             'id': video_id,
             'title': title,
@@ -55,4 +59,4 @@ class RteIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'duration': duration,
-        }
\ No newline at end of file
+        }

From 3462af03e6c8c64b916fe90ecea0ef7719438ad2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 17:59:07 +0100
Subject: [PATCH 491/974] [rte] PEP8

---
 youtube_dl/extractor/rte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index b5a67373b..04158b993 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -24,7 +24,7 @@ class RteIE(InfoExtractor):
             'skip_download': 'f4m fails with --test atm'
         }
     }
-    
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)

From c80ede5f1380ce49cf8ad26d84e4eb1aa2d7c004 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 18:03:36 +0100
Subject: [PATCH 492/974] [karaoketv] Simplify (#3853)

---
 youtube_dl/extractor/karaoketv.py | 32 +++++++++++++------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index 4d50308cc..4856200d8 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -1,12 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-import json
-import sys
-
 from .common import InfoExtractor
-from ..utils import compat_urllib_parse, ExtractorError
+from ..compat import compat_urllib_parse
+from ..utils import (
+    ExtractorError,
+    js_to_json,
+)
 
 
 class KaraoketvIE(InfoExtractor):
@@ -21,22 +21,16 @@ class KaraoketvIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-
-        # BUG: SSL23_GET_SERVER_HELLO:unknown protocol 
-        if sys.hexversion < 0x03000000:
-            raise ExtractorError("Only python 3 supported.\n")
-
-        mobj = re.match(self._VALID_URL, url)
-        
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        settings_json = compat_urllib_parse.unquote_plus(self._search_regex(r'config=(.*)', self._og_search_video_url(webpage ,video_id), ''))
-        
-        urls_info_webpage = self._download_webpage(settings_json, 'Downloading settings json')
+        page_video_url = self._og_search_video_url(webpage, video_id)
+        config_json = compat_urllib_parse.unquote_plus(self._search_regex(
+            r'config=(.*)', page_video_url, 'configuration'))
 
-        urls_info_json = json.loads(urls_info_webpage.replace('\'', '"'))
+        urls_info_json = self._download_json(
+            config_json, video_id, 'Downloading configuration',
+            transform_source=js_to_json)
 
         url = urls_info_json['playlist'][0]['url']
 
@@ -44,4 +38,4 @@ class KaraoketvIE(InfoExtractor):
             'id': video_id,
             'title': self._og_search_title(webpage),
             'url': url,
-        }
\ No newline at end of file
+        }

From 6ce08764a1faba3eeecbd2a04643f3266c8c5111 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 18:58:03 +0100
Subject: [PATCH 493/974] Credit @dinesh for rte.ie (#4015)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 8f2010803..1e4cadae5 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -101,3 +101,4 @@ Thijs Vermeir
 Joel Leclerc
 Christopher Krooss
 Ondřej Caletka
+Dinesh S

From ed3958d714d8e68b3b48697333c72ff6438002f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 00:40:46 +0600
Subject: [PATCH 494/974] [collegerama] Add extractor (#4540)

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/collegerama.py | 92 +++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 youtube_dl/extractor/collegerama.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 647e0a8c2..2f3a322a7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -70,6 +70,7 @@ from .cnn import (
     CNNArticleIE,
 )
 from .collegehumor import CollegeHumorIE
+from .collegerama import CollegeRamaIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE
diff --git a/youtube_dl/extractor/collegerama.py b/youtube_dl/extractor/collegerama.py
new file mode 100644
index 000000000..fedd48490
--- /dev/null
+++ b/youtube_dl/extractor/collegerama.py
@@ -0,0 +1,92 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
+
+
+class CollegeRamaIE(InfoExtractor):
+    _VALID_URL = r'https?://collegerama\.tudelft\.nl/Mediasite/Play/(?P<id>[\da-f]+)'
+    _TESTS = [
+        {
+            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/585a43626e544bdd97aeb71a0ec907a01d',
+            'md5': '481fda1c11f67588c0d9d8fbdced4e39',
+            'info_dict': {
+                'id': '585a43626e544bdd97aeb71a0ec907a01d',
+                'ext': 'mp4',
+                'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
+                'description': '',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 7713.088,
+                'timestamp': 1413309600,
+                'upload_date': '20141014',
+            },
+        },
+        {
+            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/86a9ea9f53e149079fbdb4202b521ed21d?catalog=fd32fd35-6c99-466c-89d4-cd3c431bc8a4',
+            'md5': 'ef1fdded95bdf19b12c5999949419c92',
+            'info_dict': {
+                'id': '86a9ea9f53e149079fbdb4202b521ed21d',
+                'ext': 'wmv',
+                'title': '64ste Vakantiecursus: Afvalwater',
+                'description': 'md5:7fd774865cc69d972f542b157c328305',
+                'duration': 10853,
+                'timestamp': 1326446400,
+                'upload_date': '20120113',
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        player_options_request = {
+            "getPlayerOptionsRequest": {
+                "ResourceId": video_id,
+                "QueryString": "",
+            }
+        }
+
+        request = compat_urllib_request.Request(
+            'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
+            json.dumps(player_options_request))
+        request.add_header('Content-Type', 'application/json')
+
+        player_options = self._download_json(request, video_id)
+
+        presentation = player_options['d']['Presentation']
+        title = presentation['Title']
+        description = presentation.get('Description')
+        thumbnail = None
+        duration = float_or_none(presentation.get('Duration'), 1000)
+        timestamp = int_or_none(presentation.get('UnixTime'), 1000)
+
+        formats = []
+        for stream in presentation['Streams']:
+            for video in stream['VideoUrls']:
+                thumbnail_url = stream.get('ThumbnailUrl')
+                if thumbnail_url:
+                    thumbnail = 'http://collegerama.tudelft.nl' + thumbnail_url
+                format_id = video['MediaType']
+                if format_id == 'SS':
+                    continue
+                formats.append({
+                    'url': video['Location'],
+                    'format_id': format_id,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
+        }

From be4a824d74add1a3b78b8244dff12f4f078f168a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 19:55:36 +0100
Subject: [PATCH 495/974] Add new option --source-address

Closes #3618, fixes #721, fixes #2481, fixes #4551, closes #1020.
---
 youtube_dl/YoutubeDL.py |  6 ++---
 youtube_dl/__init__.py  |  1 +
 youtube_dl/compat.py    | 28 +++++++++++++++++++++
 youtube_dl/options.py   | 24 ++++++++++++------
 youtube_dl/update.py    |  2 +-
 youtube_dl/utils.py     | 56 +++++++++++++++++++++++++++++++++++------
 6 files changed, 98 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4cc3ec2fb..fd2c0e044 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -211,6 +211,7 @@ class YoutubeDL(object):
                        - "warn": only emit a warning
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise
+    source_address:    (Experimental) Client-side IP address to bind to.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -1493,9 +1494,8 @@ class YoutubeDL(object):
         proxy_handler = compat_urllib_request.ProxyHandler(proxies)
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
-        https_handler = make_HTTPS_handler(
-            self.params.get('nocheckcertificate', False), debuglevel=debuglevel)
-        ydlh = YoutubeDLHandler(debuglevel=debuglevel)
+        https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
+        ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         opener = compat_urllib_request.build_opener(
             https_handler, proxy_handler, cookie_processor, ydlh)
         # Delete the default user-agent header, which would otherwise apply in
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 659a92a3b..d74a304b7 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -327,6 +327,7 @@ def _real_main(argv=None):
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
         'fixup': opts.fixup,
+        'source_address': opts.source_address,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 46d438846..44a902573 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -4,6 +4,7 @@ import getpass
 import optparse
 import os
 import re
+import socket
 import subprocess
 import sys
 
@@ -307,6 +308,32 @@ else:
     compat_kwargs = lambda kwargs: kwargs
 
 
+if sys.version_info < (2, 7):
+    def compat_socket_create_connection(address, timeout, source_address=None):
+        host, port = address
+        err = None
+        for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
+            af, socktype, proto, canonname, sa = res
+            sock = None
+            try:
+                sock = socket.socket(af, socktype, proto)
+                sock.settimeout(timeout)
+                if source_address:
+                    sock.bind(source_address)
+                sock.connect(sa)
+                return sock
+            except socket.error as _:
+                err = _
+                if sock is not None:
+                    sock.close()
+        if err is not None:
+            raise err
+        else:
+            raise error("getaddrinfo returns an empty list")
+else:
+    compat_socket_create_connection = socket.create_connection
+
+
 # Fix https://github.com/rg3/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
 def workaround_optparse_bug9161():
@@ -343,6 +370,7 @@ __all__ = [
     'compat_parse_qs',
     'compat_print',
     'compat_str',
+    'compat_socket_create_connection',
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index e5602bb3a..31351d43d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -148,14 +148,6 @@ def parseOpts(overrideArguments=None):
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,
         help='Output descriptions of all supported extractors')
-    general.add_option(
-        '--proxy', dest='proxy',
-        default=None, metavar='URL',
-        help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
-    general.add_option(
-        '--socket-timeout',
-        dest='socket_timeout', type=float, default=None,
-        help='Time to wait before giving up, in seconds')
     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',
@@ -173,6 +165,21 @@ def parseOpts(overrideArguments=None):
         default=False,
         help='Do not extract the videos of a playlist, only list them.')
 
+    network = optparse.OptionGroup(parser, 'Network Options')
+    network.add_option(
+        '--proxy', dest='proxy',
+        default=None, metavar='URL',
+        help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
+    network.add_option(
+        '--socket-timeout',
+        dest='socket_timeout', type=float, default=None, metavar='SECONDS',
+        help='Time to wait before giving up, in seconds')
+    network.add_option(
+        '--source-address',
+        metavar='IP', dest='source_address', default=None,
+        help='Client-side IP address to bind to (experimental)',
+    )
+
     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(
         '--playlist-start',
@@ -652,6 +659,7 @@ def parseOpts(overrideArguments=None):
         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
 
     parser.add_option_group(general)
+    parser.add_option_group(network)
     parser.add_option_group(selection)
     parser.add_option_group(downloader)
     parser.add_option_group(filesystem)
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 3f9c5249d..d8be4049f 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -59,7 +59,7 @@ def update_self(to_screen, verbose):
         to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 
-    https_handler = make_HTTPS_handler(False)
+    https_handler = make_HTTPS_handler({})
     opener = compat_urllib_request.build_opener(https_handler)
 
     # Check if there is a new version
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a12b0a7de..cc5f510f4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -10,6 +10,7 @@ import ctypes
 import datetime
 import email.utils
 import errno
+import functools
 import gzip
 import itertools
 import io
@@ -34,7 +35,9 @@ from .compat import (
     compat_chr,
     compat_getenv,
     compat_html_entities,
+    compat_http_client,
     compat_parse_qs,
+    compat_socket_create_connection,
     compat_str,
     compat_urllib_error,
     compat_urllib_parse,
@@ -391,13 +394,14 @@ def formatSeconds(secs):
         return '%d' % secs
 
 
-def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
+def make_HTTPS_handler(params, **kwargs):
+    opts_no_check_certificate = params.get('nocheckcertificate', False)
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
         context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
         if opts_no_check_certificate:
             context.verify_mode = ssl.CERT_NONE
         try:
-            return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+            return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
         except TypeError:
             # Python 2.7.8
             # (create_default_context present but HTTPSHandler has no context=)
@@ -420,17 +424,14 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
                 except ssl.SSLError:
                     self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
 
-        class HTTPSHandlerV3(compat_urllib_request.HTTPSHandler):
-            def https_open(self, req):
-                return self.do_open(HTTPSConnectionV3, req)
-        return HTTPSHandlerV3(**kwargs)
+        return YoutubeDLHTTPSHandler(params, https_conn_class=HTTPSConnectionV3, **kwargs)
     else:  # Python < 3.4
         context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
         context.verify_mode = (ssl.CERT_NONE
                                if opts_no_check_certificate
                                else ssl.CERT_REQUIRED)
         context.set_default_verify_paths()
-        return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+        return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 
 
 class ExtractorError(Exception):
@@ -544,6 +545,26 @@ class ContentTooShortError(Exception):
         self.expected = expected
 
 
+def _create_http_connection(ydl_handler, http_class, is_https=False, *args, **kwargs):
+    hc = http_class(*args, **kwargs)
+    source_address = ydl_handler._params.get('source_address')
+    if source_address is not None:
+        sa = (source_address, 0)
+        if hasattr(hc, 'source_address'):  # Python 2.7+
+            hc.source_address = sa
+        else:  # Python 2.6
+            def _hc_connect(self, *args, **kwargs):
+                sock = compat_socket_create_connection(
+                    (self.host, self.port), self.timeout, sa)
+                if is_https:
+                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                else:
+                    self.sock = sock
+            hc.connect = functools.partial(_hc_connect, hc)
+
+    return hc
+
+
 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
@@ -562,6 +583,15 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     public domain.
     """
 
+    def __init__(self, params, *args, **kwargs):
+        compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
+        self._params = params
+
+    def http_open(self, req):
+        return self.do_open(functools.partial(
+            _create_http_connection, self, compat_http_client.HTTPConnection),
+            req)
+
     @staticmethod
     def deflate(data):
         try:
@@ -631,6 +661,18 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     https_response = http_response
 
 
+class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
+    def __init__(self, params, https_conn_class=None, *args, **kwargs):
+        compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
+        self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
+        self._params = params
+
+    def https_open(self, req):
+        return self.do_open(functools.partial(
+            _create_http_connection, self, self._https_conn_class, True),
+            req)
+
+
 def parse_iso8601(date_str, delimiter='T'):
     """ Return a UNIX timestamp from the given date """
 

From 500b8b41c1a1c27da64def50514c247e05f6fbbe Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 20:02:02 +0100
Subject: [PATCH 496/974] [options] Add -4 and -6 options

Fixes #520, fixes #3626.
---
 youtube_dl/options.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 31351d43d..e25466c39 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -179,6 +179,16 @@ def parseOpts(overrideArguments=None):
         metavar='IP', dest='source_address', default=None,
         help='Client-side IP address to bind to (experimental)',
     )
+    network.add_option(
+        '-4', '--force-ipv4',
+        action='store_const', const='0.0.0.0', dest='source_address',
+        help='Make all connections via IPv4 (experimental)',
+    )
+    network.add_option(
+        '-6', '--force-ipv6',
+        action='store_const', const='::', dest='source_address',
+        help='Make all connections via IPv6 (experimental)',
+    )
 
     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(

From c5a59d9391d23fc2f13cd87ba27982c98caa29ff Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 20:05:28 +0100
Subject: [PATCH 497/974] [utils] Fix call to _create_http_connection

Avoid confusion over args/kwargs.
---
 youtube_dl/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index cc5f510f4..42f0b07ce 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -545,7 +545,7 @@ class ContentTooShortError(Exception):
         self.expected = expected
 
 
-def _create_http_connection(ydl_handler, http_class, is_https=False, *args, **kwargs):
+def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     hc = http_class(*args, **kwargs)
     source_address = ydl_handler._params.get('source_address')
     if source_address is not None:
@@ -589,7 +589,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
     def http_open(self, req):
         return self.do_open(functools.partial(
-            _create_http_connection, self, compat_http_client.HTTPConnection),
+            _create_http_connection, self, compat_http_client.HTTPConnection, False),
             req)
 
     @staticmethod

From 149f05c7b6adcb55b96c4ca60abc4e42d98623a7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 20:06:13 +0100
Subject: [PATCH 498/974] release 2015.01.10.1

---
 README.md             | 16 ++++++++++++----
 youtube_dl/version.py |  2 +-
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 87aaec8a3..c56d906b5 100644
--- a/README.md
+++ b/README.md
@@ -60,10 +60,6 @@ which means you can modify it, redistribute it or use it however you like.
                                      they would handle
     --extractor-descriptions         Output descriptions of all supported
                                      extractors
-    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
-                                     an empty string (--proxy "") for direct
-                                     connection
-    --socket-timeout None            Time to wait before giving up, in seconds
     --default-search PREFIX          Use this prefix for unqualified URLs. For
                                      example "gvsearch2:" downloads two videos
                                      from google videos for  youtube-dl "large
@@ -82,6 +78,18 @@ which means you can modify it, redistribute it or use it however you like.
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them.
 
+## Network Options:
+    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
+                                     an empty string (--proxy "") for direct
+                                     connection
+    --socket-timeout SECONDS         Time to wait before giving up, in seconds
+    --source-address IP              Client-side IP address to bind to
+                                     (experimental)
+    -4, --force-ipv4                 Make all connections via IPv4
+                                     (experimental)
+    -6, --force-ipv6                 Make all connections via IPv6
+                                     (experimental)
+
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 60258d36a..267ffeeb4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.10'
+__version__ = '2015.01.10.1'

From 58b1f00d19d608470540b111f3825eefafbdae97 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 21:02:27 +0100
Subject: [PATCH 499/974] [YoutubeDL] Add new --call-home option for debugging

---
 youtube_dl/YoutubeDL.py | 14 ++++++++++++++
 youtube_dl/__init__.py  |  1 +
 youtube_dl/options.py   |  4 ++++
 3 files changed, 19 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fd2c0e044..84cea472c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -58,6 +58,7 @@ from .utils import (
     takewhile_inclusive,
     UnavailableVideoError,
     url_basename,
+    version_tuple,
     write_json_file,
     write_string,
     YoutubeDLHandler,
@@ -212,6 +213,8 @@ class YoutubeDL(object):
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise
     source_address:    (Experimental) Client-side IP address to bind to.
+    call_home:         (Experimental) Boolean, true iff we are allowed to
+                       contact the youtube-dl servers for debugging.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -1464,6 +1467,17 @@ class YoutubeDL(object):
                 proxy_map.update(handler.proxies)
         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
 
+        if self.params.get('call_home', False):
+            ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
+            self._write_string('[debug] Public IP address: %s\n' % ipaddr)
+            latest_version = self.urlopen(
+                'https://yt-dl.org/latest/version').read().decode('utf-8')
+            if version_tuple(latest_version) > version_tuple(__version__):
+                self.report_warning(
+                    'You are using an outdated version (newest version: %s)! '
+                    'See https://yt-dl.org/update if you need help updating.' %
+                    latest_version)
+
     def _setup_opener(self):
         timeout_val = self.params.get('socket_timeout')
         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index d74a304b7..ddf6260d1 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -328,6 +328,7 @@ def _real_main(argv=None):
         'postprocessors': postprocessors,
         'fixup': opts.fixup,
         'source_address': opts.source_address,
+        'call_home': opts.call_home,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index e25466c39..0176bae75 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -505,6 +505,10 @@ def parseOpts(overrideArguments=None):
         '--print-traffic',
         dest='debug_printtraffic', action='store_true', default=False,
         help='Display sent and read HTTP traffic')
+    verbosity.add_option(
+        '-C', '--call-home',
+        dest='call_home', action='store_true', default=False,
+        help='Contact the youtube-dl server for debugging. (Experimental)')
 
     filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
     filesystem.add_option(

From cd3063f3fa81f1ca30ff9994b7346ad7ebadd0fe Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 21:03:00 +0100
Subject: [PATCH 500/974] release 2015.01.10.2

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c56d906b5..3113f70ee 100644
--- a/README.md
+++ b/README.md
@@ -239,6 +239,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      files in the current directory to debug
                                      problems
     --print-traffic                  Display sent and read HTTP traffic
+    -C, --call-home                  Contact the youtube-dl server for
+                                     debugging. (Experimental)
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 267ffeeb4..e5e320c76 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.10.1'
+__version__ = '2015.01.10.2'

From c071733fd441109633bb9bb57007fae754b7610d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 21:07:44 +0100
Subject: [PATCH 501/974] [README] Highlight that bug reports should include
 the -v output

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3113f70ee..006e8d709 100644
--- a/README.md
+++ b/README.md
@@ -627,7 +627,9 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode.
 
-Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+**Please include the full output of youtube-dl when run with `-v`**.
+
+The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
 Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 

From 8bfa75451b6506db324e766ef5d815d10eee5619 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 21:09:15 +0100
Subject: [PATCH 502/974] [options] Add --no-call-home

While we're at it, also drop "experimental" moniker for --call-home - should work fine.
---
 youtube_dl/YoutubeDL.py | 4 ++--
 youtube_dl/options.py   | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 84cea472c..47e143d89 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -213,8 +213,8 @@ class YoutubeDL(object):
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise
     source_address:    (Experimental) Client-side IP address to bind to.
-    call_home:         (Experimental) Boolean, true iff we are allowed to
-                       contact the youtube-dl servers for debugging.
+    call_home:         Boolean, true iff we are allowed to contact the
+                       youtube-dl servers for debugging.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0176bae75..a30974efd 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -508,7 +508,11 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '-C', '--call-home',
         dest='call_home', action='store_true', default=False,
-        help='Contact the youtube-dl server for debugging. (Experimental)')
+        help='Contact the youtube-dl server for debugging.')
+    verbosity.add_option(
+        '--no-call-home',
+        dest='call_home', action='store_false', default=False,
+        help='Do NOT contact the youtube-dl server for debugging.')
 
     filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
     filesystem.add_option(

From 987493aef37cce273a10ddfbe540a41a4c6a67cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 15:13:03 +0600
Subject: [PATCH 503/974] [test_compat] Fix alphabetic order to make
 test_all_present pass

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 44a902573..07ceaa170 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -369,8 +369,8 @@ __all__ = [
     'compat_ord',
     'compat_parse_qs',
     'compat_print',
-    'compat_str',
     'compat_socket_create_connection',
+    'compat_str',
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',

From d156a1d981b2773e1c35c9eb7d9b9a3bbcea7c96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 15:25:29 +0600
Subject: [PATCH 504/974] [xboxclips] Fix extraction

---
 youtube_dl/extractor/xboxclips.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xboxclips.py b/youtube_dl/extractor/xboxclips.py
index 9cf867807..236ff403b 100644
--- a/youtube_dl/extractor/xboxclips.py
+++ b/youtube_dl/extractor/xboxclips.py
@@ -30,7 +30,7 @@ class XboxClipsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._html_search_regex(
-            r'>(?:Link|Download): <a href="([^"]+)">', webpage, 'video URL')
+            r'>(?:Link|Download): <a[^>]+href="([^"]+)"', webpage, 'video URL')
         title = self._html_search_regex(
             r'<title>XboxClips \| ([^<]+)</title>', webpage, 'title')
         upload_date = unified_strdate(self._html_search_regex(

From d5bb814d348558a9700f9d8d2a063a2de2de4274 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 15:42:53 +0600
Subject: [PATCH 505/974] [veehd] Capture removed video message

---
 youtube_dl/extractor/veehd.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 815f58468..4f76ca403 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -8,6 +8,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    ExtractorError,
     clean_html,
     get_element_by_id,
 )
@@ -34,6 +35,10 @@ class VeeHDIE(InfoExtractor):
         # See https://github.com/rg3/youtube-dl/issues/2102
         self._download_webpage(url, video_id, 'Requesting webpage')
         webpage = self._download_webpage(url, video_id)
+
+        if 'This video has been removed<' in webpage:
+            raise ExtractorError('Video %s has been removed' % video_id, expected=True)
+
         player_path = self._search_regex(
             r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
             webpage, 'player path')

From 8ad6b5ed9f90e07fb2847b38818d1352c71fcb18 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 10:47:39 +0100
Subject: [PATCH 506/974] [compat] Correct socket error class reference

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 07ceaa170..4453b34fc 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -329,7 +329,7 @@ if sys.version_info < (2, 7):
         if err is not None:
             raise err
         else:
-            raise error("getaddrinfo returns an empty list")
+            raise socket.error("getaddrinfo returns an empty list")
 else:
     compat_socket_create_connection = socket.create_connection
 

From 6a5fa754903370c8e522cb3c85fd1952c1d9ff9c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 10:48:20 +0100
Subject: [PATCH 507/974] [karaoketv] Remove unused import

---
 youtube_dl/extractor/karaoketv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index 4856200d8..e3b43ff8d 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
 from ..utils import (
-    ExtractorError,
     js_to_json,
 )
 

From a798e64c1549f4bdf240dec4492203b304a39783 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 16:20:16 +0600
Subject: [PATCH 508/974] [veehd] Improve extraction

---
 youtube_dl/extractor/veehd.py | 37 +++++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 4f76ca403..5dc38dd94 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -47,18 +47,35 @@ class VeeHDIE(InfoExtractor):
         self._download_webpage(player_url, video_id, 'Requesting player page')
         player_page = self._download_webpage(
             player_url, video_id, 'Downloading player page')
-        config_json = self._search_regex(
-            r'value=\'config=({.+?})\'', player_page, 'config json')
-        config = json.loads(config_json)
 
-        video_url = compat_urlparse.unquote(config['clip']['url'])
+        config_json = self._search_regex(
+            r'value=\'config=({.+?})\'', player_page, 'config json', default=None)
+
+        if config_json:
+            config = json.loads(config_json)
+            video_url = compat_urlparse.unquote(config['clip']['url'])
+        else:
+            iframe_src = self._search_regex(
+                r'<iframe[^>]+src="/?([^"]+)"', player_page, 'iframe url')
+            iframe_url = 'http://veehd.com/%s' % iframe_src
+
+            self._download_webpage(iframe_url, video_id, 'Requesting iframe page')
+            iframe_page = self._download_webpage(
+                iframe_url, video_id, 'Downloading iframe page')
+
+            video_url = self._search_regex(
+                r"file\s*:\s*'([^']+)'", iframe_page, 'video url')
+
         title = clean_html(get_element_by_id('videoName', webpage).rpartition('|')[0])
-        uploader_id = self._html_search_regex(r'<a href="/profile/\d+">(.+?)</a>',
-                                              webpage, 'uploader')
-        thumbnail = self._search_regex(r'<img id="veehdpreview" src="(.+?)"',
-                                       webpage, 'thumbnail')
-        description = self._html_search_regex(r'<td class="infodropdown".*?<div>(.*?)<ul',
-                                              webpage, 'description', flags=re.DOTALL)
+        uploader_id = self._html_search_regex(
+            r'<a href="/profile/\d+">(.+?)</a>',
+            webpage, 'uploader')
+        thumbnail = self._search_regex(
+            r'<img id="veehdpreview" src="(.+?)"',
+            webpage, 'thumbnail')
+        description = self._html_search_regex(
+            r'<td class="infodropdown".*?<div>(.*?)<ul',
+            webpage, 'description', flags=re.DOTALL)
 
         return {
             '_type': 'video',

From aa24de39aae6b9a6ad59720258b9018f544a888f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 16:20:39 +0600
Subject: [PATCH 509/974] [veehd] Update test

---
 youtube_dl/extractor/veehd.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 5dc38dd94..96353f525 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -18,13 +18,13 @@ class VeeHDIE(InfoExtractor):
     _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://veehd.com/video/4686958',
+        'url': 'http://veehd.com/video/4639434_Solar-Sinter',
         'info_dict': {
-            'id': '4686958',
+            'id': '4639434',
             'ext': 'mp4',
-            'title': 'Time Lapse View from Space ( ISS)',
-            'uploader_id': 'spotted',
-            'description': 'md5:f0094c4cf3a72e22bc4e4239ef767ad7',
+            'title': 'Solar Sinter',
+            'uploader_id': 'VideoEyes',
+            'description': 'md5:46a840e8692ddbaffb5f81d9885cb457',
         },
     }
 

From f3ff1a3696c4080468e2cc5810c34273b148bd3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 23 Feb 2014 11:29:42 +0100
Subject: [PATCH 510/974] YoutubeDL: Make the decision about removing the
 original file after each postprocessor is run (fixes #2261)

If one of the processors said the file should be kept, it wouldn't pay
attention to the response from the following processors. This was wrong if the
'keep_video' option was False, if the first extractor modifies the original file
and then we extract its audio we don't want to keep the original video file.
---
 youtube_dl/YoutubeDL.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 47e143d89..65581ec63 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1235,14 +1235,15 @@ class YoutubeDL(object):
         """Run all the postprocessors on the given file."""
         info = dict(ie_info)
         info['filepath'] = filename
-        keep_video = None
         pps_chain = []
         if ie_info.get('__postprocessors') is not None:
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:
+            keep_video = None
+            old_filename = info['filepath']
             try:
-                keep_video_wish, new_info = pp.run(info)
+                keep_video_wish, info = pp.run(info)
                 if keep_video_wish is not None:
                     if keep_video_wish:
                         keep_video = keep_video_wish
@@ -1251,12 +1252,12 @@ class YoutubeDL(object):
                         keep_video = keep_video_wish
             except PostProcessingError as e:
                 self.report_error(e.msg)
-        if keep_video is False and not self.params.get('keepvideo', False):
-            try:
-                self.to_screen('Deleting original file %s (pass -k to keep)' % filename)
-                os.remove(encodeFilename(filename))
-            except (IOError, OSError):
-                self.report_warning('Unable to remove downloaded video file')
+            if keep_video is False and not self.params.get('keepvideo', False):
+                try:
+                    self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
+                    os.remove(encodeFilename(old_filename))
+                except (IOError, OSError):
+                    self.report_warning('Unable to remove downloaded video file')
 
     def _make_archive_id(self, info_dict):
         # Future-proof against any change in case

From 4340decad25e264863e27538847e17918b4c1ae2 Mon Sep 17 00:00:00 2001
From: Pierre <p.mdawar@hotmail.com>
Date: Thu, 25 Sep 2014 19:37:20 +0300
Subject: [PATCH 511/974] check for overwriting files in the downloader (fixes
 #3916, closes #3829)

---
 youtube_dl/YoutubeDL.py         | 81 ++++++++++++++++-----------------
 youtube_dl/downloader/common.py | 13 +++++-
 2 files changed, 51 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 65581ec63..772fddd45 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1121,49 +1121,46 @@ class YoutubeDL(object):
                                             (info_dict['thumbnail'], compat_str(err)))
 
         if not self.params.get('skip_download', False):
-            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
-                success = True
-            else:
-                try:
-                    def dl(name, info):
-                        fd = get_suitable_downloader(info)(self, self.params)
-                        for ph in self._progress_hooks:
-                            fd.add_progress_hook(ph)
-                        if self.params.get('verbose'):
-                            self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
-                        return fd.download(name, info)
-                    if info_dict.get('requested_formats') is not None:
-                        downloaded = []
-                        success = True
-                        merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
-                        if not merger._executable:
-                            postprocessors = []
-                            self.report_warning('You have requested multiple '
-                                                'formats but ffmpeg or avconv are not installed.'
-                                                ' The formats won\'t be merged')
-                        else:
-                            postprocessors = [merger]
-                        for f in info_dict['requested_formats']:
-                            new_info = dict(info_dict)
-                            new_info.update(f)
-                            fname = self.prepare_filename(new_info)
-                            fname = prepend_extension(fname, 'f%s' % f['format_id'])
-                            downloaded.append(fname)
-                            partial_success = dl(fname, new_info)
-                            success = success and partial_success
-                        info_dict['__postprocessors'] = postprocessors
-                        info_dict['__files_to_merge'] = downloaded
+            try:
+                def dl(name, info):
+                    fd = get_suitable_downloader(info)(self, self.params)
+                    for ph in self._progress_hooks:
+                        fd.add_progress_hook(ph)
+                    if self.params.get('verbose'):
+                        self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
+                    return fd.download(name, info)
+                if info_dict.get('requested_formats') is not None:
+                    downloaded = []
+                    success = True
+                    merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
+                    if not merger._executable:
+                        postprocessors = []
+                        self.report_warning('You have requested multiple '
+                                            'formats but ffmpeg or avconv are not installed.'
+                                            ' The formats won\'t be merged')
                     else:
-                        # Just a single file
-                        success = dl(filename, info_dict)
-                except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                    self.report_error('unable to download video data: %s' % str(err))
-                    return
-                except (OSError, IOError) as err:
-                    raise UnavailableVideoError(err)
-                except (ContentTooShortError, ) as err:
-                    self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
-                    return
+                        postprocessors = [merger]
+                    for f in info_dict['requested_formats']:
+                        new_info = dict(info_dict)
+                        new_info.update(f)
+                        fname = self.prepare_filename(new_info)
+                        fname = prepend_extension(fname, 'f%s' % f['format_id'])
+                        downloaded.append(fname)
+                        partial_success = dl(fname, new_info)
+                        success = success and partial_success
+                    info_dict['__postprocessors'] = postprocessors
+                    info_dict['__files_to_merge'] = downloaded
+                else:
+                    # Just a single file
+                    success = dl(filename, info_dict)
+            except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                self.report_error('unable to download video data: %s' % str(err))
+                return
+            except (OSError, IOError) as err:
+                raise UnavailableVideoError(err)
+            except (ContentTooShortError, ) as err:
+                self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+                return
 
             if success:
                 # Fixup content
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 584bde732..de6b9311d 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -284,8 +284,19 @@ class FileDownloader(object):
         """Download to a filename using the info from info_dict
         Return True on success and False otherwise
         """
+        nooverwrites_and_exists = (
+            self.params.get('nooverwrites', False)
+            and os.path.exists(encodeFilename(filename))
+        )
+
+        continuedl_and_exists = (
+            self.params.get('continuedl', False)
+            and os.path.isfile(encodeFilename(filename))
+            and not self.params.get('nopart', False)
+        )
+
         # Check file already present
-        if filename != '-' and self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
+        if filename != '-' and nooverwrites_and_exists or continuedl_and_exists:
             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,

From 6ccbb335d2af17a0882a728ced155e3b00271044 Mon Sep 17 00:00:00 2001
From: Johan <Josso@users.noreply.github.com>
Date: Sun, 11 Jan 2015 17:39:16 +0100
Subject: [PATCH 512/974] [drtv] Updated with support for https

---
 youtube_dl/extractor/drtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 93b3c9f36..c44adb109 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -6,7 +6,7 @@ from ..utils import parse_iso8601
 
 
 class DRTVIE(SubtitlesInfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)+(?P<id>[\da-z-]+)(?:[/#?]|$)'
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)+(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
     _TEST = {
         'url': 'http://www.dr.dk/tv/se/partiets-mand/partiets-mand-7-8',

From 2aeb06d6dc8c615cf98b7d21ad152bb393726b8a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 17:40:45 +0100
Subject: [PATCH 513/974] [utils] Improve colon handling (Fixes #4683)

---
 test/test_utils.py  | 4 ++++
 youtube_dl/utils.py | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 16e1a1ddf..5e16f30a2 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -79,6 +79,10 @@ class TestUtil(unittest.TestCase):
         tests = '\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430'
         self.assertEqual(sanitize_filename(tests), tests)
 
+        self.assertEqual(
+            sanitize_filename('New World record at 0:12:34'),
+            'New World record at 0_12_34')
+
         forbidden = '"\0\\/'
         for fc in forbidden:
             for fbc in forbidden:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 42f0b07ce..deaf71803 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -287,6 +287,8 @@ def sanitize_filename(s, restricted=False, is_id=False):
             return '_'
         return char
 
+    # Handle timestamps
+    s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
     result = ''.join(map(replace_insane, s))
     if not is_id:
         while '__' in result:

From 15aecd871147bcec28b341b334b2e37eaf6cef04 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 17:47:04 +0100
Subject: [PATCH 514/974] release 2015.01.11

---
 README.md             | 4 +++-
 youtube_dl/version.py | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 006e8d709..c26546f2a 100644
--- a/README.md
+++ b/README.md
@@ -240,7 +240,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      problems
     --print-traffic                  Display sent and read HTTP traffic
     -C, --call-home                  Contact the youtube-dl server for
-                                     debugging. (Experimental)
+                                     debugging.
+    --no-call-home                   Do NOT contact the youtube-dl server for
+                                     debugging.
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e5e320c76..0a184ad08 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.10.2'
+__version__ = '2015.01.11'

From a0977064ce9e79fd55f9da904ae3ae0a502b46c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 23:18:45 +0600
Subject: [PATCH 515/974] [npo] Fix non asf streams (Closes #4680)

---
 youtube_dl/extractor/npo.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 8da76ae45..47802518d 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -10,6 +10,7 @@ from ..utils import (
     strip_jsonp,
     url_basename,
     fix_xml_ampersands,
+    determine_ext,
 )
 
 
@@ -125,6 +126,12 @@ class NPOIE(InfoExtractor):
                 stream_url = stream.get('url')
                 if not stream_url:
                     continue
+                if determine_ext(stream_url).lower() != 'asf':
+                    formats.append({
+                        'url': stream_url,
+                        'quality': stream.get('kwaliteit'),
+                    })
+                    continue
                 asx = self._download_xml(
                     stream_url, video_id,
                     'Downloading stream %d ASX playlist' % i,

From c85f368370e3e72f65fa1a40f6b6f785b4b2faf6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 23:45:52 +0600
Subject: [PATCH 516/974] [npo] Make extension check less strict and add test
 (#4680)

---
 youtube_dl/extractor/npo.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 47802518d..fcd4fea94 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -68,6 +68,16 @@ class NPOIE(InfoExtractor):
                 'skip_download': True,
             }
         },
+        # non asf in streams
+        {
+            'url': 'http://www.npo.nl/hoe-gaat-europa-verder-na-parijs/10-01-2015/WO_NOS_762771',
+            'md5': 'b3da13de374cbe2d5332a7e910bef97f',
+            'info_dict': {
+                'id': 'WO_NOS_762771',
+                'ext': 'mp4',
+                'title': 'Hoe gaat Europa verder na Parijs?',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -126,7 +136,7 @@ class NPOIE(InfoExtractor):
                 stream_url = stream.get('url')
                 if not stream_url:
                     continue
-                if determine_ext(stream_url).lower() != 'asf':
+                if not '.asf' in stream_url:
                     formats.append({
                         'url': stream_url,
                         'quality': stream.get('kwaliteit'),

From 8e7a9016d5e79679f4b948f00c794b8ff31a41e4 Mon Sep 17 00:00:00 2001
From: "Johan K. Jensen" <jj@johanjensen.dk>
Date: Sun, 11 Jan 2015 23:23:10 +0100
Subject: [PATCH 517/974] [DRBonanza] Add new extractor (fixing #4581)

---
 youtube_dl/extractor/__init__.py  |   1 +
 youtube_dl/extractor/drbonanza.py | 120 ++++++++++++++++++++++++++++++
 2 files changed, 121 insertions(+)
 create mode 100644 youtube_dl/extractor/drbonanza.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2f3a322a7..5a1c2f69b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -93,6 +93,7 @@ from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
 from .dotsub import DotsubIE
 from .dreisat import DreiSatIE
+from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
 from .drtv import DRTVIE
 from .dvtv import DVTVIE
diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dl/extractor/drbonanza.py
new file mode 100644
index 000000000..8cbdb57b3
--- /dev/null
+++ b/youtube_dl/extractor/drbonanza.py
@@ -0,0 +1,120 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .common import ExtractorError
+from ..utils import parse_iso8601
+import json
+import re
+
+class DRBonanzaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/bonanza/(?:[^/]+/)+(?:[^/])+?(?:assetId=(?P<id>\d+))?(?:[#&]|$)'
+
+    _TESTS = [{
+        'url': 'http://www.dr.dk/bonanza/serie/portraetter/Talkshowet.htm?assetId=65517',
+        'md5': 'fe330252ddea607635cf2eb2c99a0af3',
+        'info_dict': {
+            'id': '65517',
+            'ext': 'mp4',
+            'title': 'Talkshowet - Leonard Cohen',
+            'description': 'md5:8f34194fb30cd8c8a30ad8b27b70c0ca',
+            'timestamp': 1295537932,
+            'upload_date': '20110120',
+            'duration': 3664000,
+        },
+    },{
+        'url': 'http://www.dr.dk/bonanza/radio/serie/sport/fodbold.htm?assetId=59410',
+        'md5': '6dfe039417e76795fb783c52da3de11d',
+        'info_dict': {
+            'id': '59410',
+            'ext': 'mp3',
+            'title': 'EM fodbold 1992 Danmark - Tyskland finale Transmission',
+            'description': 'md5:501e5a195749480552e214fbbed16c4e',
+            'timestamp': 1223274900,
+            'upload_date': '20081006',
+            'duration': 7369000,
+        },
+    }]
+
+    def _real_extract(self, url):
+        url_id = self._match_id(url)
+        
+        webpage = self._download_webpage(url, url_id if url_id else "")
+        
+        if url_id:
+            info = json.loads(self._html_search_regex(r'({.*?' + url_id + '.*})', webpage, 'json'))
+        else:
+            # Just fetch the first video on that page
+            info = json.loads(self._html_search_regex(r'bonanzaFunctions.newPlaylist\(({.*})\)', webpage, 'json'))
+        
+        asset_id = str(info['AssetId'])
+        title = info['Title'].rstrip(' \'\"-,.:;!?')
+        duration = info['Duration']
+        timestamp = parse_iso8601(re.sub(r'\.\d+$', '', info['Created'])) # First published online. "FirstPublished" contains the date for original airing.
+        
+        def parse_filename_info(url):
+            match = re.search(r'/\d+_(?P<width>\d+)x(?P<height>\d+)x(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
+            if match:
+                return {'width': int(match.group(1)), 'height': int(match.group(2)), 'bitrate': int(match.group(3)), 'ext': match.group(4)}
+            match = re.search(r'/\d+_(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
+            if match:
+                return {'bitrate': int(match.group(1)), 'ext': match.group(2)}
+            return {'width': None, 'height': None, 'bitrate': None, 'ext': None}
+        
+        video_types = ['VideoHigh', 'VideoMid', 'VideoLow']
+        preferencemap = {
+            'VideoHigh': -1,
+            'VideoMid': -2,
+            'VideoLow': -3,
+            'Audio': -4,
+        }
+        
+        formats = []
+        for file in info['Files']:
+            if info['Type'] == "Video":
+                if file['Type'] in video_types:
+                    fileinfo = parse_filename_info(file['Location'])
+                    formats.append({
+                        'url': file['Location'],
+                        'format_id': file['Type'].replace('Video', ''),
+                        'preference': preferencemap.get(file['Type'], -10),
+                        'width': fileinfo['width'],
+                        'height': fileinfo['height'],
+                        'vbr': fileinfo['bitrate'],
+                        'ext': fileinfo['ext'],
+                    })
+                elif file['Type'] == "Thumb":
+                    thumbnail = file['Location']
+            elif info['Type'] == "Audio":
+                if file['Type'] == "Audio":
+                    fileinfo = parse_filename_info(file['Location'])
+                    formats.append({
+                        'url': file['Location'],
+                        'format_id': file['Type'],
+                        'abr': fileinfo['bitrate'],
+                        'ext': fileinfo['ext'],
+                        'vcodec': 'none',
+                    })
+                elif file['Type'] == "Thumb":
+                    thumbnail = file['Location']
+        
+        description = "{}\n{}\n{}\n".format(info['Description'], info['Actors'], info['Colophon'])
+
+        for f in formats:
+            f['url'] = f['url'].replace('rtmp://vod-bonanza.gss.dr.dk/bonanza/', 'http://vodfiles.dr.dk/')
+            f['url'] = f['url'].replace('mp4:bonanza', 'bonanza')
+        
+        self._sort_formats(formats)
+        
+        display_id = re.sub(r'[^\w\d-]', '', re.sub(r' ', '-', title.lower())) + '-' + asset_id
+        display_id = re.sub(r'-+', '-', display_id)
+        
+        return {
+            'id': asset_id,
+            'display_id': display_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+        }

From 8e2ec95575fc8157da1ec3ec408737f200aa44d7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 23:40:27 +0100
Subject: [PATCH 518/974] [drbonanza] Simplify and fix duration (#4687)

---
 youtube_dl/extractor/drbonanza.py | 79 ++++++++++++++++++-------------
 1 file changed, 45 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dl/extractor/drbonanza.py
index 8cbdb57b3..7626219ba 100644
--- a/youtube_dl/extractor/drbonanza.py
+++ b/youtube_dl/extractor/drbonanza.py
@@ -1,11 +1,15 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from .common import ExtractorError
-from ..utils import parse_iso8601
 import json
 import re
 
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
+
+
 class DRBonanzaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dr\.dk/bonanza/(?:[^/]+/)+(?:[^/])+?(?:assetId=(?P<id>\d+))?(?:[#&]|$)'
 
@@ -17,11 +21,12 @@ class DRBonanzaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Talkshowet - Leonard Cohen',
             'description': 'md5:8f34194fb30cd8c8a30ad8b27b70c0ca',
+            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
             'timestamp': 1295537932,
             'upload_date': '20110120',
-            'duration': 3664000,
+            'duration': 3664,
         },
-    },{
+    }, {
         'url': 'http://www.dr.dk/bonanza/radio/serie/sport/fodbold.htm?assetId=59410',
         'md5': '6dfe039417e76795fb783c52da3de11d',
         'info_dict': {
@@ -29,37 +34,47 @@ class DRBonanzaIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'EM fodbold 1992 Danmark - Tyskland finale Transmission',
             'description': 'md5:501e5a195749480552e214fbbed16c4e',
+            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
             'timestamp': 1223274900,
             'upload_date': '20081006',
-            'duration': 7369000,
+            'duration': 7369,
         },
     }]
 
     def _real_extract(self, url):
         url_id = self._match_id(url)
-        
-        webpage = self._download_webpage(url, url_id if url_id else "")
-        
+        webpage = self._download_webpage(url, url_id)
+
         if url_id:
-            info = json.loads(self._html_search_regex(r'({.*?' + url_id + '.*})', webpage, 'json'))
+            info = json.loads(self._html_search_regex(r'({.*?%s.*})' % url_id, webpage, 'json'))
         else:
             # Just fetch the first video on that page
             info = json.loads(self._html_search_regex(r'bonanzaFunctions.newPlaylist\(({.*})\)', webpage, 'json'))
-        
+
         asset_id = str(info['AssetId'])
         title = info['Title'].rstrip(' \'\"-,.:;!?')
-        duration = info['Duration']
-        timestamp = parse_iso8601(re.sub(r'\.\d+$', '', info['Created'])) # First published online. "FirstPublished" contains the date for original airing.
-        
+        duration = int_or_none(info.get('Duration'), scale=1000)
+        # First published online. "FirstPublished" contains the date for original airing.
+        timestamp = parse_iso8601(
+            re.sub(r'\.\d+$', '', info['Created']))
+
         def parse_filename_info(url):
             match = re.search(r'/\d+_(?P<width>\d+)x(?P<height>\d+)x(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
             if match:
-                return {'width': int(match.group(1)), 'height': int(match.group(2)), 'bitrate': int(match.group(3)), 'ext': match.group(4)}
+                return {
+                    'width': int(match.group('width')),
+                    'height': int(match.group('height')),
+                    'vbr': int(match.group('bitrate')),
+                    'ext': match.group('ext')
+                }
             match = re.search(r'/\d+_(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
             if match:
-                return {'bitrate': int(match.group(1)), 'ext': match.group(2)}
-            return {'width': None, 'height': None, 'bitrate': None, 'ext': None}
-        
+                return {
+                    'vbr': int(match.group('bitrate')),
+                    'ext': match.group(2)
+                }
+            return {}
+
         video_types = ['VideoHigh', 'VideoMid', 'VideoLow']
         preferencemap = {
             'VideoHigh': -1,
@@ -67,47 +82,43 @@ class DRBonanzaIE(InfoExtractor):
             'VideoLow': -3,
             'Audio': -4,
         }
-        
+
         formats = []
         for file in info['Files']:
             if info['Type'] == "Video":
                 if file['Type'] in video_types:
-                    fileinfo = parse_filename_info(file['Location'])
-                    formats.append({
+                    format = parse_filename_info(file['Location'])
+                    format.update({
                         'url': file['Location'],
                         'format_id': file['Type'].replace('Video', ''),
                         'preference': preferencemap.get(file['Type'], -10),
-                        'width': fileinfo['width'],
-                        'height': fileinfo['height'],
-                        'vbr': fileinfo['bitrate'],
-                        'ext': fileinfo['ext'],
                     })
+                    formats.append(format)
                 elif file['Type'] == "Thumb":
                     thumbnail = file['Location']
             elif info['Type'] == "Audio":
                 if file['Type'] == "Audio":
-                    fileinfo = parse_filename_info(file['Location'])
-                    formats.append({
+                    format = parse_filename_info(file['Location'])
+                    format.update({
                         'url': file['Location'],
                         'format_id': file['Type'],
-                        'abr': fileinfo['bitrate'],
-                        'ext': fileinfo['ext'],
                         'vcodec': 'none',
                     })
+                    formats.append(format)
                 elif file['Type'] == "Thumb":
                     thumbnail = file['Location']
-        
-        description = "{}\n{}\n{}\n".format(info['Description'], info['Actors'], info['Colophon'])
+
+        description = '%s\n%s\n%s\n' % (
+            info['Description'], info['Actors'], info['Colophon'])
 
         for f in formats:
             f['url'] = f['url'].replace('rtmp://vod-bonanza.gss.dr.dk/bonanza/', 'http://vodfiles.dr.dk/')
             f['url'] = f['url'].replace('mp4:bonanza', 'bonanza')
-        
         self._sort_formats(formats)
-        
+
         display_id = re.sub(r'[^\w\d-]', '', re.sub(r' ', '-', title.lower())) + '-' + asset_id
         display_id = re.sub(r'-+', '-', display_id)
-        
+
         return {
             'id': asset_id,
             'display_id': display_id,

From 263255eb8d45377e54e23841b55f0609f0e7e8a4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 23:42:24 +0100
Subject: [PATCH 519/974] Credit @Josso for drbonanza (#4581)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 1e4cadae5..35e256d49 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -102,3 +102,4 @@ Joel Leclerc
 Christopher Krooss
 Ondřej Caletka
 Dinesh S
+Johan K. Jensen

From 5c4a81d9345f0520a9d40acbb8212fb94bcc9e24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 23:43:09 +0100
Subject: [PATCH 520/974] [npo] Remove unused import

---
 youtube_dl/extractor/npo.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index fcd4fea94..6d20f755a 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -4,13 +4,12 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    unified_strdate,
+    fix_xml_ampersands,
     parse_duration,
     qualities,
     strip_jsonp,
+    unified_strdate,
     url_basename,
-    fix_xml_ampersands,
-    determine_ext,
 )
 
 
@@ -136,7 +135,7 @@ class NPOIE(InfoExtractor):
                 stream_url = stream.get('url')
                 if not stream_url:
                     continue
-                if not '.asf' in stream_url:
+                if '.asf' not in stream_url:
                     formats.append({
                         'url': stream_url,
                         'quality': stream.get('kwaliteit'),

From 227d4822fff13ea449f11531ce54c56b2b01432b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jan 2015 21:06:26 +0600
Subject: [PATCH 521/974] [utils] Disallow non string types in parse_duration
 (Closes #4679)

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index deaf71803..98732e8e9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1269,7 +1269,7 @@ def float_or_none(v, scale=1, invscale=1, default=None):
 
 
 def parse_duration(s):
-    if s is None:
+    if not isinstance(s, basestring if sys.version_info < (3, 0) else compat_str):
         return None
 
     s = s.strip()

From a5fb718c50d394aeee827f6e9381eceb839128de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jan 2015 21:39:58 +0600
Subject: [PATCH 522/974] [test_utils] Add more tests for parse_duration

---
 test/test_utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 5e16f30a2..399cdfdbd 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -212,6 +212,8 @@ class TestUtil(unittest.TestCase):
 
     def test_parse_duration(self):
         self.assertEqual(parse_duration(None), None)
+        self.assertEqual(parse_duration(False), None)
+        self.assertEqual(parse_duration('invalid'), None)
         self.assertEqual(parse_duration('1'), 1)
         self.assertEqual(parse_duration('1337:12'), 80232)
         self.assertEqual(parse_duration('9:12:43'), 33163)

From 3d5f7a3947a8d304bc7ad46217f171996e95c475 Mon Sep 17 00:00:00 2001
From: Johannes Knoedtel <johannes@dev-baron.de>
Date: Mon, 12 Jan 2015 22:26:20 +0100
Subject: [PATCH 523/974] [utils] Prevent override of custom headers.

The dict of headers of request objects in urllib has its keys always
capitalized.

This causes the lookup to fail and overwrite the header. If for example
a Extractor tries to add a "User-Agent" header the internal
representation in the request object is "User-agent". The header is
therefore clobbered by the "User-Agent" in std_headers, because the
strings are not equal.
---
 youtube_dl/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 98732e8e9..daf94abd1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -611,7 +611,9 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
     def http_request(self, req):
         for h, v in std_headers.items():
-            if h not in req.headers:
+            # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
+            # The dict keys are capitalized because of this bug by urllib
+            if h.capitalize() not in req.headers:
                 req.add_header(h, v)
         if 'Youtubedl-no-compression' in req.headers:
             if 'Accept-encoding' in req.headers:

From fbef83f399b6a54380cecfce6d00964cc1e063d0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 13 Jan 2015 08:10:08 +0100
Subject: [PATCH 524/974] [README] Add FAQ for playing from another machine
 (Fixes #4693)

---
 README.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index c26546f2a..078e9df82 100644
--- a/README.md
+++ b/README.md
@@ -444,9 +444,15 @@ Apparently YouTube requires you to pass a CAPTCHA test if you download too much.
 
 Once the video is fully downloaded, use any video player, such as [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).
 
-### The links provided by youtube-dl -g are not working anymore
+### I extracted a video URL with -g, but it does not play on another machine / in my webbrowser.
 
-The URLs youtube-dl outputs require the downloader to have the correct cookies. Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
+It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
+
+It may be beneficial to use IPv6; in some cases, the restrictions are only applied to IPv4. Some services (sometimes only for a subset of videos) do not restrict the video URL by IP address, cookie, or user-agent, but these are the exception rather than the rule.
+
+Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using -g, your own downloader must support these as well.
+
+If you want to play the video on a machine that is not running youtube-dl, you can relay the video content from the machine that runs youtube-dl. You can use `-o -` to let youtube-dl stream a video to stdout, or simply allow the player to download the files written by youtube-dl in turn.
 
 ### ERROR: no fmt_url_map or conn information found in video info
 

From 04e0bac233fd4f094983d612d6c76b5590393230 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 13 Jan 2015 20:54:03 +0600
Subject: [PATCH 525/974] [npo:live] Add extractor (Closes #4691)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/npo.py      | 99 +++++++++++++++++++++++++++++---
 2 files changed, 91 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5a1c2f69b..2d31c298f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -295,6 +295,7 @@ from .nowness import NownessIE
 from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
+    NPOLiveIE,
     TegenlichtVproIE,
 )
 from .nrk import (
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 6d20f755a..8d496488b 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -13,7 +13,16 @@ from ..utils import (
 )
 
 
-class NPOIE(InfoExtractor):
+class NPOBaseIE(InfoExtractor):
+    def _get_token(self, video_id):
+        token_page = self._download_webpage(
+            'http://ida.omroep.nl/npoplayer/i.js',
+            video_id, note='Downloading token')
+        return self._search_regex(
+            r'npoplayer\.token = "(.+?)"', token_page, 'token')
+
+
+class NPOIE(NPOBaseIE):
     IE_NAME = 'npo.nl'
     _VALID_URL = r'https?://www\.npo\.nl/[^/]+/[^/]+/(?P<id>[^/?]+)'
 
@@ -80,8 +89,7 @@ class NPOIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         return self._get_info(video_id)
 
     def _get_info(self, video_id):
@@ -91,12 +99,8 @@ class NPOIE(InfoExtractor):
             # We have to remove the javascript callback
             transform_source=strip_jsonp,
         )
-        token_page = self._download_webpage(
-            'http://ida.omroep.nl/npoplayer/i.js',
-            video_id,
-            note='Downloading token'
-        )
-        token = self._search_regex(r'npoplayer\.token = "(.+?)"', token_page, 'token')
+
+        token = self._get_token(video_id)
 
         formats = []
 
@@ -170,6 +174,83 @@ class NPOIE(InfoExtractor):
         }
 
 
+class NPOLiveIE(NPOBaseIE):
+    IE_NAME = 'npo.nl:live'
+    _VALID_URL = r'https?://www\.npo\.nl/live/(?P<id>.+)'
+
+    _TEST = {
+        'url': 'http://www.npo.nl/live/npo-1',
+        'info_dict': {
+            'id': 'LI_NEDERLAND1_136692',
+            'display_id': 'npo-1',
+            'ext': 'mp4',
+            'title': 're:^Nederland 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'Livestream',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        live_id = self._search_regex(
+            r'data-prid="([^"]+)"', webpage, 'live id')
+
+        metadata = self._download_json(
+            'http://e.omroep.nl/metadata/%s' % live_id,
+            display_id, transform_source=strip_jsonp)
+
+        token = self._get_token(display_id)
+
+        formats = []
+
+        streams = metadata.get('streams')
+        if streams:
+            for stream in streams:
+                stream_type = stream.get('type').lower()
+                if stream_type == 'ss':
+                    continue
+                stream_info = self._download_json(
+                    'http://ida.omroep.nl/aapi/?stream=%s&token=%s&type=jsonp'
+                    % (stream.get('url'), token),
+                    display_id, 'Downloading %s JSON' % stream_type)
+                if stream_info.get('error_code', 0) or stream_info.get('errorcode', 0):
+                    continue
+                stream_url = self._download_json(
+                    stream_info['stream'], display_id,
+                    'Downloading %s URL' % stream_type,
+                    transform_source=strip_jsonp)
+                if stream_type == 'hds':
+                    f4m_formats = self._extract_f4m_formats(stream_url, display_id)
+                    # f4m downloader downloads only piece of live stream
+                    for f4m_format in f4m_formats:
+                        f4m_format['preference'] = -1
+                    formats.extend(f4m_formats)
+                elif stream_type == 'hls':
+                    formats.extend(self._extract_m3u8_formats(stream_url, display_id, 'mp4'))
+                else:
+                    formats.append({
+                        'url': stream_url,
+                    })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': live_id,
+            'display_id': display_id,
+            'title': self._live_title(metadata['titel']),
+            'description': metadata['info'],
+            'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
+            'formats': formats,
+            'is_live': True,
+        }
+
+
 class TegenlichtVproIE(NPOIE):
     IE_NAME = 'tegenlicht.vpro.nl'
     _VALID_URL = r'https?://tegenlicht\.vpro\.nl/afleveringen/.*?'

From 034206cec1903dc8c5829012e7130a63dc014265 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 14 Jan 2015 00:14:59 +0200
Subject: [PATCH 526/974] [lnkgo] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/lnkgo.py    | 122 +++++++++++++++++++++++++++++++
 2 files changed, 123 insertions(+)
 create mode 100644 youtube_dl/extractor/lnkgo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2d31c298f..0902eb437 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -226,6 +226,7 @@ from .livestream import (
     LivestreamOriginalIE,
     LivestreamShortenerIE,
 )
+from .lnkgo import LnkGoIE
 from .lrt import LRTIE
 from .lynda import (
     LyndaIE,
diff --git a/youtube_dl/extractor/lnkgo.py b/youtube_dl/extractor/lnkgo.py
new file mode 100644
index 000000000..cfae60154
--- /dev/null
+++ b/youtube_dl/extractor/lnkgo.py
@@ -0,0 +1,122 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    unified_strdate,
+)
+
+
+class LnkGoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?lnkgo\.alfa\.lt/visi\-video/(?P<show>[^/]+)/ziurek\-(?P<display_id>[A-Za-z0-9\-]+)'
+    _TESTS = [{
+        'url': 'http://lnkgo.alfa.lt/visi-video/yra-kaip-yra/ziurek-yra-kaip-yra-162',
+        'info_dict': {
+            'id': '46712',
+            'ext': 'mp4',
+            'title': 'Yra kaip yra',
+            'upload_date': '20150107',
+            'description': 'md5:d82a5e36b775b7048617f263a0e3475e',
+            'age_limit': 7,
+            'duration': 3019,
+            'thumbnail': 're:^https?://.*\.jpg$'
+        },
+        'params': {
+            'skip_download': True,  # HLS download
+        },
+    }, {
+        'url': 'http://lnkgo.alfa.lt/visi-video/aktualai-pratesimas/ziurek-nerdas-taiso-kompiuteri-2',
+        'info_dict': {
+            'id': '47289',
+            'ext': 'mp4',
+            'title': 'Nėrdas: Kompiuterio Valymas',
+            'upload_date': '20150113',
+            'description': 'md5:7352d113a242a808676ff17e69db6a69',
+            'age_limit': 18,
+            'duration': 346,
+            'thumbnail': 're:^https?://.*\.jpg$'
+        },
+        'params': {
+            'skip_download': True,  # HLS download
+        },
+    }]
+    _AGE_LIMITS = {
+        'N-7': 7,
+        'N-14': 14,
+        'S': 18,
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(
+            url, display_id, 'Downloading player webpage')
+
+        video_id = self._search_regex(
+            r'data-ep="([^"]+)"', webpage, 'video ID')
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+
+        thumbnail = self._og_search_thumbnail(webpage)
+        thumbnail_w = int_or_none(
+            self._og_search_property('image:width', webpage, 'thumbnail width', fatal=False))
+        thumbnail_h = int_or_none(
+            self._og_search_property('image:height', webpage, 'thumbnail height', fatal=False))
+        thumbnails = [{
+            'url': thumbnail,
+            'width': thumbnail_w,
+            'height': thumbnail_h,
+        }]
+
+        upload_date = unified_strdate(self._search_regex(
+            r'class="meta-item\sair-time">.*?<strong>([^<]+)</strong>', webpage, 'upload date', fatal=False))
+        duration = int_or_none(self._search_regex(
+            r'VideoDuration = "([^"]+)"', webpage, 'duration', fatal=False))
+
+        pg_rating = self._search_regex(
+            r'pgrating="([^"]+)"', webpage, 'PG rating', fatal=False, default='')
+        age_limit = self._AGE_LIMITS.get(pg_rating, 0)
+
+        sources_js = self._search_regex(
+            r'(?s)sources:\s(\[.*?\]),', webpage, 'sources')
+        sources = self._parse_json(
+            sources_js, video_id, transform_source=js_to_json)
+
+        formats = []
+        for source in sources:
+            if source.get('provider') == 'rtmp':
+                m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<play_path>.+)$', source['file'])
+                if not m:
+                    continue
+                formats.append({
+                    'format_id': 'rtmp',
+                    'ext': 'flv',
+                    'url': m.group('url'),
+                    'play_path': m.group('play_path'),
+                    'page_url': url,
+                })
+            elif source.get('file').endswith('.m3u8'):
+                formats.append({
+                    'format_id': 'hls',
+                    'ext': source.get('type', 'mp4'),
+                    'url': source['file'],
+                })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'duration': duration,
+            'description': description,
+            'age_limit': age_limit,
+            'upload_date': upload_date,
+        }

From a69801e2c6f53985acf475f391f7f7205775d631 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 14 Jan 2015 00:16:34 +0200
Subject: [PATCH 527/974] [utils] Add additional format to unified_strdate

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 399cdfdbd..206760d99 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -148,6 +148,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_strdate('8/7/2009'), '20090708')
         self.assertEqual(unified_strdate('Dec 14, 2012'), '20121214')
         self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011')
+        self.assertEqual(unified_strdate('1968 12 10'), '19681210')
         self.assertEqual(unified_strdate('1968-12-10'), '19681210')
         self.assertEqual(unified_strdate('28/01/2014 21:00:00 +0100'), '20140128')
         self.assertEqual(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 98732e8e9..38c18c127 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -721,6 +721,7 @@ def unified_strdate(date_str, day_first=True):
         '%b %dst %Y %I:%M%p',
         '%b %dnd %Y %I:%M%p',
         '%b %dth %Y %I:%M%p',
+        '%Y %m %d',
         '%Y-%m-%d',
         '%Y/%m/%d',
         '%Y/%m/%d %H:%M:%S',

From f2cbc96c3ea3f0c3b447b50b445de736e535d63f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 14 Jan 2015 00:51:48 +0200
Subject: [PATCH 528/974] [lnkgo] Make more robust

---
 youtube_dl/extractor/lnkgo.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/lnkgo.py b/youtube_dl/extractor/lnkgo.py
index cfae60154..a8e357859 100644
--- a/youtube_dl/extractor/lnkgo.py
+++ b/youtube_dl/extractor/lnkgo.py
@@ -62,16 +62,18 @@ class LnkGoIE(InfoExtractor):
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)
 
-        thumbnail = self._og_search_thumbnail(webpage)
         thumbnail_w = int_or_none(
             self._og_search_property('image:width', webpage, 'thumbnail width', fatal=False))
         thumbnail_h = int_or_none(
             self._og_search_property('image:height', webpage, 'thumbnail height', fatal=False))
-        thumbnails = [{
-            'url': thumbnail,
-            'width': thumbnail_w,
-            'height': thumbnail_h,
-        }]
+        thumbnail = {
+            'url': self._og_search_thumbnail(webpage),
+        }
+        if thumbnail_w and thumbnail_h:
+            thumbnail.update({
+                'width': thumbnail_w,
+                'height': thumbnail_h,
+            })
 
         upload_date = unified_strdate(self._search_regex(
             r'class="meta-item\sair-time">.*?<strong>([^<]+)</strong>', webpage, 'upload date', fatal=False))
@@ -80,7 +82,7 @@ class LnkGoIE(InfoExtractor):
 
         pg_rating = self._search_regex(
             r'pgrating="([^"]+)"', webpage, 'PG rating', fatal=False, default='')
-        age_limit = self._AGE_LIMITS.get(pg_rating, 0)
+        age_limit = self._AGE_LIMITS.get(pg_rating.upper(), 0)
 
         sources_js = self._search_regex(
             r'(?s)sources:\s(\[.*?\]),', webpage, 'sources')
@@ -114,7 +116,7 @@ class LnkGoIE(InfoExtractor):
             'display_id': display_id,
             'title': title,
             'formats': formats,
-            'thumbnails': thumbnails,
+            'thumbnails': [thumbnail],
             'duration': duration,
             'description': description,
             'age_limit': age_limit,

From e92d4a11f570abb47215b13f88020f52479766e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Jan 2015 20:27:14 +0600
Subject: [PATCH 529/974] [spiegel] Test format video URLs for 404 (Closes
 #4579)

---
 youtube_dl/extractor/spiegel.py | 49 ++++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 1e55a9ffb..f345883c7 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -4,7 +4,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_HTTPError,
+)
+from ..utils import (
+    HEADRequest,
+    ExtractorError,
+)
 from .spiegeltv import SpiegeltvIE
 
 
@@ -60,21 +67,31 @@ class SpiegelIE(InfoExtractor):
         xml_url = base_url + video_id + '.xml'
         idoc = self._download_xml(xml_url, video_id)
 
-        formats = [
-            {
-                'format_id': n.tag.rpartition('type')[2],
-                'url': base_url + n.find('./filename').text,
-                'width': int(n.find('./width').text),
-                'height': int(n.find('./height').text),
-                'abr': int(n.find('./audiobitrate').text),
-                'vbr': int(n.find('./videobitrate').text),
-                'vcodec': n.find('./codec').text,
-                'acodec': 'MP4A',
-            }
-            for n in list(idoc)
-            # Blacklist type 6, it's extremely LQ and not available on the same server
-            if n.tag.startswith('type') and n.tag != 'type6'
-        ]
+        formats = []
+        for n in list(idoc):
+            if n.tag.startswith('type') and n.tag != 'type6':
+                format_id = n.tag.rpartition('type')[2]
+                video_url = base_url + n.find('./filename').text
+                # Test video URLs beforehand as some of them are invalid
+                try:
+                    self._request_webpage(
+                        HEADRequest(video_url), video_id,
+                        'Checking %s video URL' % format_id)
+                except ExtractorError as e:
+                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                        self.report_warning(
+                            '%s video URL is invalid, skipping' % format_id, video_id)
+                        continue
+                formats.append({
+                    'format_id': format_id,
+                    'url': video_url,
+                    'width': int(n.find('./width').text),
+                    'height': int(n.find('./height').text),
+                    'abr': int(n.find('./audiobitrate').text),
+                    'vbr': int(n.find('./videobitrate').text),
+                    'vcodec': n.find('./codec').text,
+                    'acodec': 'MP4A',
+                })
         duration = float(idoc[0].findall('./duration')[0].text)
 
         self._sort_formats(formats)

From c9ef44ce296ded2d5ba4b36ad60b0a7add7944bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Jan 2015 21:50:36 +0600
Subject: [PATCH 530/974] [smotri] Improve extraction (Closes #4698)

---
 youtube_dl/extractor/smotri.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index baef3daa0..26f361c93 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -90,6 +90,20 @@ class SmotriIE(InfoExtractor):
             },
             'skip': 'Video is not approved by moderator',
         },
+        # not approved by moderator, but available
+        {
+            'url': 'http://smotri.com/video/view/?id=v28888533b73',
+            'md5': 'f44bc7adac90af518ef1ecf04893bb34',
+            'info_dict': {
+                'id': 'v28888533b73',
+                'ext': 'mp4',
+                'title': 'Russian Spies Killed By ISIL Child Soldier',
+                'uploader': 'Mopeder',
+                'uploader_id': 'mopeder',
+                'duration': 71,
+                'thumbnail': 'http://frame9.loadup.ru/d7/32/2888853.2.3.jpg',
+            },
+        },
         # swf player
         {
             'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',
@@ -146,13 +160,16 @@ class SmotriIE(InfoExtractor):
 
         video = self._download_json(request, video_id, 'Downloading video JSON')
 
-        if video.get('_moderate_no') or not video.get('moderated'):
-            raise ExtractorError('Video %s has not been approved by moderator' % video_id, expected=True)
-
-        if video.get('error'):
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
         video_url = video.get('_vidURL') or video.get('_vidURL_mp4')
+
+        if not video_url:
+            if video.get('_moderate_no') or not video.get('moderated'):
+                raise ExtractorError(
+                    'Video %s has not been approved by moderator' % video_id, expected=True)
+
+            if video.get('error'):
+                raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
         title = video['title']
         thumbnail = video['_imgURL']
         upload_date = unified_strdate(video['added'])

From be5f2c192cde200b6fcc8d754fde8b5e479a300c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 15 Jan 2015 02:06:48 +0100
Subject: [PATCH 531/974] [ssl] Correct connect creation

We want to authenticate the server, see https://docs.python.org/dev/library/ssl.html#ssl.Purpose.SERVER_AUTH .
---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 38c18c127..2546fa45d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -399,8 +399,9 @@ def formatSeconds(secs):
 def make_HTTPS_handler(params, **kwargs):
     opts_no_check_certificate = params.get('nocheckcertificate', False)
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
-        context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+        context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
         if opts_no_check_certificate:
+            context.check_hostname = False
             context.verify_mode = ssl.CERT_NONE
         try:
             return YoutubeDLHTTPSHandler(params, context=context, **kwargs)

From 4cfaf85c65d1b00ee5b198acb44f1ac65366e0d8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 15 Jan 2015 12:42:11 +0100
Subject: [PATCH 532/974] release 2015.01.15

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0a184ad08..6eccf2bf5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.11'
+__version__ = '2015.01.15'

From c8dfe360eb642b2957116814bfdef67686ab000d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Jan 2015 21:43:35 +0600
Subject: [PATCH 533/974] [atresplayer] Add authentication support (Closes
 #4700)

---
 youtube_dl/extractor/atresplayer.py | 32 ++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 72e83bfc2..5db1941b3 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -4,9 +4,12 @@ import time
 import hmac
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
+    compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     int_or_none,
     float_or_none,
     xpath_text,
@@ -44,6 +47,33 @@ class AtresPlayerIE(InfoExtractor):
     _PLAYER_URL_TEMPLATE = 'https://servicios.atresplayer.com/episode/getplayer.json?episodePk=%s'
     _EPISODE_URL_TEMPLATE = 'http://www.atresplayer.com/episodexml/%s'
 
+    _LOGIN_URL = 'https://servicios.atresplayer.com/j_spring_security_check'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'j_username': username,
+            'j_password': password,
+        }
+
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        error = self._html_search_regex(
+            r'(?s)<ul class="list_error">(.+?)</ul>', response, 'error', default=None)
+        if error:
+            raise ExtractorError(
+                'Unable to login: %s' % error, expected=True)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From a45c0a5d67b87e9ea16e2812f44753c9cb946636 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 15 Jan 2015 19:57:36 +0200
Subject: [PATCH 534/974] [videomega] Fix extraction (Closes #4703)

---
 youtube_dl/extractor/videomega.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 7a78f0d26..9fc64d172 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_request,
 )
 from ..utils import (
     remove_start,
@@ -16,20 +17,23 @@ class VideoMegaIE(InfoExtractor):
         (?:iframe\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
         '''
     _TEST = {
-        'url': 'http://videomega.tv/?ref=GKeGPVedBe',
-        'md5': '240fb5bcf9199961f48eb17839b084d6',
+        'url': 'http://videomega.tv/?ref=QR0HCUHI1661IHUCH0RQ',
+        'md5': 'bf5c2f95c4c917536e80936af7bc51e1',
         'info_dict': {
-            'id': 'GKeGPVedBe',
+            'id': 'QR0HCUHI1661IHUCH0RQ',
             'ext': 'mp4',
-            'title': 'XXL - All Sports United',
+            'title': 'Big Buck Bunny',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'http://videomega.tv/iframe.php?ref={0:}'.format(video_id)
-        webpage = self._download_webpage(url, video_id)
+
+        iframe_url = 'http://videomega.tv/iframe.php?ref={0:}'.format(video_id)
+        req = compat_urllib_request.Request(iframe_url)
+        req.add_header('Referer', url)
+        webpage = self._download_webpage(req, video_id)
 
         escaped_data = self._search_regex(
             r'unescape\("([^"]+)"\)', webpage, 'escaped data')
@@ -37,13 +41,13 @@ class VideoMegaIE(InfoExtractor):
 
         thumbnail = self._search_regex(
             r'image:\s*"([^"]+)"', playlist, 'thumbnail', fatal=False)
-        url = self._search_regex(r'file:\s*"([^"]+)"', playlist, 'URL')
+        video_url = self._search_regex(r'file:\s*"([^"]+)"', playlist, 'URL')
         title = remove_start(self._html_search_regex(
             r'<title>(.*?)</title>', webpage, 'title'), 'VideoMega.tv - ')
 
         formats = [{
             'format_id': 'sd',
-            'url': url,
+            'url': video_url,
         }]
         self._sort_formats(formats)
 
@@ -52,4 +56,5 @@ class VideoMegaIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
+            'http_referer': iframe_url,
         }

From a9b6b5cd15cff004b4109ab64b181fff23c10770 Mon Sep 17 00:00:00 2001
From: Phil Kulak <pkulak@gmail.com>
Date: Thu, 15 Jan 2015 10:23:05 -0800
Subject: [PATCH 535/974] Looks like Google switched to a new JS compiler that
 includes dollar signs in function names.

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c7611a3a0..073b78fb5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -588,7 +588,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
-            r'\.sig\|\|([a-zA-Z0-9]+)\(', jscode,
+            r'\.sig\|\|([a-zA-Z0-9\$]+)\(', jscode,
             'Initial JS player signature function name')
 
         jsi = JSInterpreter(jscode)

From aaeb86f68289496595930a7d6a04457917826d13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 16 Jan 2015 01:25:03 +0600
Subject: [PATCH 536/974] [youtube] Add test for #4706

---
 youtube_dl/extractor/youtube.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 073b78fb5..2a1f8be0a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -394,6 +394,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'format': '141',
             },
         },
+        # JS player signature function name containing $
+        {
+            'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
+            'info_dict': {
+                'id': 'nfWlot6h_JM',
+                'ext': 'm4a',
+                'title': 'Taylor Swift - Shake It Off',
+                'description': 'md5:2acfda1b285bdd478ccec22f9918199d',
+                'uploader': 'TaylorSwiftVEVO',
+                'uploader_id': 'TaylorSwiftVEVO',
+                'upload_date': '20140818',
+            },
+            'params': {
+                'youtube_include_dash_manifest': True,
+                'format': '141',
+            },
+        },
         # Controversy video
         {
             'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
@@ -588,7 +605,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
-            r'\.sig\|\|([a-zA-Z0-9\$]+)\(', jscode,
+            r'\.sig\|\|([a-zA-Z0-9$]+)\(', jscode,
             'Initial JS player signature function name')
 
         jsi = JSInterpreter(jscode)

From 31d4a6e2121ab5954aec5cfb48d6b1f1cbffefbd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 15 Jan 2015 22:38:11 +0100
Subject: [PATCH 537/974] release 2015.01.15.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6eccf2bf5..458813b2d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.15'
+__version__ = '2015.01.15.1'

From e205db3bcd7f7a9b0beaa13a4774bdcb26091e2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 16 Jan 2015 13:29:01 +0100
Subject: [PATCH 538/974] FFmpegEmbedSubtitlePP: don't fail if the video
 doesn't have an audio stream (fixes #4718)

Instead of specifying which streams ffmpeg must copy, we tell it to copy all.
---
 youtube_dl/postprocessor/ffmpeg.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 26b99e43c..a75277778 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -475,7 +475,13 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         filename = information['filepath']
         input_files = [filename] + [subtitles_filename(filename, lang, self._subformat) for lang in sub_langs]
 
-        opts = ['-map', '0:0', '-map', '0:1', '-c:v', 'copy', '-c:a', 'copy']
+        opts = [
+            '-map', '0',
+            '-c', 'copy',
+            # Don't copy the existing subtitles, we may be running the
+            # postprocessor a second time
+            '-map', '-0:s',
+        ]
         for (i, lang) in enumerate(sub_langs):
             opts.extend(['-map', '%d:0' % (i + 1), '-c:s:%d' % i, 'mov_text'])
             lang_code = self._conver_lang_code(lang)

From 2875cf01bb2229cb6cceeac72fb6dd770f7a96d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 16 Jan 2015 13:37:37 +0100
Subject: [PATCH 539/974] FFmpegEmbedSubtitlePP: simplify command

---
 youtube_dl/postprocessor/ffmpeg.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index a75277778..5b0ff32b1 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -481,15 +481,15 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
             # Don't copy the existing subtitles, we may be running the
             # postprocessor a second time
             '-map', '-0:s',
+            '-c:s', 'mov_text',
         ]
         for (i, lang) in enumerate(sub_langs):
-            opts.extend(['-map', '%d:0' % (i + 1), '-c:s:%d' % i, 'mov_text'])
+            opts.extend(['-map', '%d:0' % (i + 1)])
             lang_code = self._conver_lang_code(lang)
             if lang_code is not None:
                 opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
-        opts.extend(['-f', 'mp4'])
 
-        temp_filename = filename + '.temp'
+        temp_filename = prepend_extension(filename, 'temp')
         self._downloader.to_screen('[ffmpeg] Embedding subtitles in \'%s\'' % filename)
         self.run_ffmpeg_multiple_files(input_files, temp_filename, opts)
         os.remove(encodeFilename(filename))

From 4f4f6428224b641eb97ffd4b90be9d67034fc64a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 16 Jan 2015 13:44:36 +0100
Subject: [PATCH 540/974] [npo] Remove unused import

---
 youtube_dl/extractor/npo.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 8d496488b..175b14583 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     fix_xml_ampersands,

From 9d22a7dfb07ad6ff23680e11238744b2bb1735aa Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 16 Jan 2015 13:44:44 +0100
Subject: [PATCH 541/974] [fourtube] Fix extraction

---
 youtube_dl/extractor/fourtube.py | 76 ++++++++++++++++++--------------
 youtube_dl/utils.py              |  2 +-
 2 files changed, 45 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 7187e0752..b2284ab01 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -7,10 +7,9 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
-    clean_html,
     parse_duration,
+    parse_iso8601,
     str_to_int,
-    unified_strdate,
 )
 
 
@@ -28,68 +27,81 @@ class FourTubeIE(InfoExtractor):
             'uploader': 'WCP Club',
             'uploader_id': 'wcp-club',
             'upload_date': '20131031',
+            'timestamp': 1383263892,
             'duration': 583,
+            'view_count': int,
+            'like_count': int,
+            'categories': list,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage_url = 'http://www.4tube.com/videos/' + video_id
-        webpage = self._download_webpage(webpage_url, video_id)
+        webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
+        title = self._html_search_meta('name', webpage)
+        timestamp = parse_iso8601(self._html_search_meta(
+            'uploadDate', webpage))
+        thumbnail = self._html_search_meta('thumbnailUrl', webpage)
+        uploader_id = self._html_search_regex(
+            r'<a class="img-avatar" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
+            webpage, 'uploader id')
+        uploader = self._html_search_regex(
+            r'<a class="img-avatar" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
+            webpage, 'uploader')
 
-        playlist_json = self._html_search_regex(r'var playerConfigPlaylist\s+=\s+([^;]+)', webpage, 'Playlist')
-        media_id = self._search_regex(r'idMedia:\s*(\d+)', playlist_json, 'Media Id')
-        sources = self._search_regex(r'sources:\s*\[([^\]]*)\]', playlist_json, 'Sources').split(',')
-        title = self._search_regex(r'title:\s*"([^"]*)', playlist_json, 'Title')
-        thumbnail_url = self._search_regex(r'image:\s*"([^"]*)', playlist_json, 'Thumbnail', fatal=False)
+        categories_html = self._search_regex(
+            r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="list">(.*?)</ul>',
+            webpage, 'categories', fatal=False)
+        categories = None
+        if categories_html:
+            categories = [
+                c.strip() for c in re.findall(
+                    r'(?s)<li><a.*?>(.*?)</a>', categories_html)]
 
-        uploader_str = self._search_regex(r'<span>Uploaded by</span>(.*?)<span>', webpage, 'uploader', fatal=False)
-        mobj = re.search(r'<a href="/sites/(?P<id>[^"]+)"><strong>(?P<name>[^<]+)</strong></a>', uploader_str)
-        (uploader, uploader_id) = (mobj.group('name'), mobj.group('id')) if mobj else (clean_html(uploader_str), None)
+        view_count = str_to_int(self._search_regex(
+            r'<meta itemprop="interactionCount" content="UserPlays:([0-9,]+)">',
+            webpage, 'view count', fatal=False))
+        like_count = str_to_int(self._search_regex(
+            r'<meta itemprop="interactionCount" content="UserLikes:([0-9,]+)">',
+            webpage, 'like count', fatal=False))
+        duration = parse_duration(self._html_search_meta('duration', webpage))
 
-        upload_date = None
-        view_count = None
-        duration = None
-        description = self._html_search_meta('description', webpage, 'description')
-        if description:
-            upload_date = self._search_regex(r'Published Date: (\d{2} [a-zA-Z]{3} \d{4})', description, 'upload date',
-                                             fatal=False)
-            if upload_date:
-                upload_date = unified_strdate(upload_date)
-            view_count = self._search_regex(r'Views: ([\d,\.]+)', description, 'view count', fatal=False)
-            if view_count:
-                view_count = str_to_int(view_count)
-            duration = parse_duration(self._search_regex(r'Length: (\d+m\d+s)', description, 'duration', fatal=False))
+        params_js = self._search_regex(
+            r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
+            webpage, 'initialization parameters'
+        )
+        params = self._parse_json('[%s]' % params_js, video_id)
+        media_id = params[0]
+        sources = ['%s' % p for p in params[2]]
 
-        token_url = "http://tkn.4tube.com/{0}/desktop/{1}".format(media_id, "+".join(sources))
+        token_url = 'http://tkn.4tube.com/{0}/desktop/{1}'.format(
+            media_id, '+'.join(sources))
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
             b'Origin': b'http://www.4tube.com',
         }
         token_req = compat_urllib_request.Request(token_url, b'{}', headers)
         tokens = self._download_json(token_req, video_id)
-
         formats = [{
             'url': tokens[format]['token'],
             'format_id': format + 'p',
             'resolution': format + 'p',
             'quality': int(format),
         } for format in sources]
-
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
-            'thumbnail': thumbnail_url,
+            'categories': categories,
+            'thumbnail': thumbnail,
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
+            'like_count': like_count,
             'view_count': view_count,
             'duration': duration,
             'age_limit': 18,
-            'webpage_url': webpage_url,
         }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2546fa45d..7832ed87f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1277,7 +1277,7 @@ def parse_duration(s):
     s = s.strip()
 
     m = re.match(
-        r'''(?ix)T?
+        r'''(?ix)(?:P?T)?
         (?:
             (?P<only_mins>[0-9.]+)\s*(?:mins?|minutes?)\s*|
             (?P<only_hours>[0-9.]+)\s*(?:hours?)|

From cce81f192c15f3766adb7340315a877bc4fc592b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 16 Jan 2015 14:20:25 +0100
Subject: [PATCH 542/974] [bandcamp:album] Fix title extraction (Fixes #4721)

---
 youtube_dl/extractor/bandcamp.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index b45d68a61..aea0263d6 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -161,7 +161,8 @@ class BandcampAlbumIE(InfoExtractor):
         entries = [
             self.url_result(compat_urlparse.urljoin(url, t_path), ie=BandcampIE.ie_key())
             for t_path in tracks_paths]
-        title = self._search_regex(r'album_title : "(.*?)"', webpage, 'title')
+        title = self._search_regex(
+            r'album_title\s*:\s*"(.*?)"', webpage, 'title', fatal=False)
         return {
             '_type': 'playlist',
             'id': playlist_id,

From 910c55205293097e79b2a134b69b572984e37846 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 16 Jan 2015 14:20:38 +0100
Subject: [PATCH 543/974] release 2015.01.16

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 458813b2d..63a79a7ee 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.15.1'
+__version__ = '2015.01.16'

From ba319696a99f34342a321d23bb41d3d442a773cc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 17 Jan 2015 23:56:34 +0100
Subject: [PATCH 544/974] [options] Clarify that --password can be left out
 (#4723)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a30974efd..f25c12e52 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -264,7 +264,7 @@ def parseOpts(overrideArguments=None):
     authentication.add_option(
         '-p', '--password',
         dest='password', metavar='PASSWORD',
-        help='account password')
+        help='account password. If this option is left out, youtube-dl will ask interactively.')
     authentication.add_option(
         '-2', '--twofactor',
         dest='twofactor', metavar='TWOFACTOR',

From fdb2ed7455d509a5f0b17ad5c1d721d5484bde8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Jan 2015 08:09:18 +0600
Subject: [PATCH 545/974] [abc7news] Add extractor (Closes #4734)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/abc7news.py | 68 ++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 youtube_dl/extractor/abc7news.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0902eb437..2b9d4455d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .abc import ABCIE
+from .abc7news import Abc7NewsIE
 from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
 from .adobetv import AdobeTVIE
diff --git a/youtube_dl/extractor/abc7news.py b/youtube_dl/extractor/abc7news.py
new file mode 100644
index 000000000..c04949c21
--- /dev/null
+++ b/youtube_dl/extractor/abc7news.py
@@ -0,0 +1,68 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601
+
+
+class Abc7NewsIE(InfoExtractor):
+    _VALID_URL = r'https?://abc7news\.com(?:/[^/]+/(?P<display_id>[^/]+))?/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://abc7news.com/entertainment/east-bay-museum-celebrates-vintage-synthesizers/472581/',
+            'info_dict': {
+                'id': '472581',
+                'display_id': 'east-bay-museum-celebrates-vintage-synthesizers',
+                'ext': 'mp4',
+                'title': 'East Bay museum celebrates history of synthesized music',
+                'description': 'md5:a4f10fb2f2a02565c1749d4adbab4b10',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'timestamp': 1421123075,
+                'upload_date': '20150113',
+                'uploader': 'Jonathan Bloom',
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://abc7news.com/472581',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        webpage = self._download_webpage(url, display_id)
+
+        m3u8 = self._html_search_meta(
+            'contentURL', webpage, 'm3u8 url', fatal=True)
+
+        formats = self._extract_m3u8_formats(m3u8, display_id, 'mp4')
+        self._sort_formats(formats)
+
+        title = self._og_search_title(webpage).strip()
+        description = self._og_search_description(webpage).strip()
+        thumbnail = self._og_search_thumbnail(webpage)
+        timestamp = parse_iso8601(self._search_regex(
+            r'<div class="meta">\s*<time class="timeago" datetime="([^"]+)">',
+            webpage, 'upload date', fatal=False))
+        uploader = self._search_regex(
+            r'rel="author">([^<]+)</a>',
+            webpage, 'uploader', default=None)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'formats': formats,
+        }

From bd3cbe0716c3cfc4c6ce7a5a338e881a06b19ef8 Mon Sep 17 00:00:00 2001
From: David Powell <bitlooter@users.noreply.github.com>
Date: Sun, 18 Jan 2015 15:01:05 -0800
Subject: [PATCH 546/974] Embed description and URL as MP4 tags

---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 5b0ff32b1..faccdc43d 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -509,6 +509,10 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             metadata['artist'] = info['uploader']
         elif info.get('uploader_id') is not None:
             metadata['artist'] = info['uploader_id']
+        if info.get('description') is not None:
+            metadata['description'] = info['description']
+        if info.get('webpage_url') is not None:
+            metadata['comment'] = info['webpage_url']
 
         if not metadata:
             self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')

From c511f13f223d843dd4affc30c76bf333a6863227 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 19 Jan 2015 10:10:05 +0100
Subject: [PATCH 547/974] [ndtv] Modernize

---
 youtube_dl/extractor/ndtv.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ndtv.py b/youtube_dl/extractor/ndtv.py
index 95e7d63aa..2a1ca80df 100644
--- a/youtube_dl/extractor/ndtv.py
+++ b/youtube_dl/extractor/ndtv.py
@@ -27,9 +27,7 @@ class NDTVIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         filename = self._search_regex(

From 030aa5d9e7f482ab7baba18a9ff45e41021d60e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Jan 2015 23:00:22 +0600
Subject: [PATCH 548/974] [tvp] Fix extraction

---
 youtube_dl/extractor/tvp.py | 37 +++++++++++++++++--------------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index cc26f417a..f57d609d4 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -12,61 +12,59 @@ class TvpIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem/wideo/odc-2/4278035',
+        'md5': 'cdd98303338b8a7f7abab5cd14092bf2',
         'info_dict': {
             'id': '4278035',
             'ext': 'wmv',
             'title': 'Ogniem i mieczem, odc. 2',
-            'description': 'Bohun dowiaduje się o złamaniu przez kniahinię danego mu słowa i wyrusza do Rozłogów. Helenie w ostatniej chwili udaje się uciec dzięki pomocy Zagłoby.',
         },
     }, {
         'url': 'http://vod.tvp.pl/seriale/obyczajowe/czas-honoru/sezon-1-1-13/i-seria-odc-13/194536',
+        'md5': '8aa518c15e5cc32dfe8db400dc921fbb',
         'info_dict': {
             'id': '194536',
             'ext': 'mp4',
             'title': 'Czas honoru, I seria – odc. 13',
-            #  'description': 'WŁADEK\nCzesław prosi Marię o dostarczenie Władkowi zarazki tyfusu. Jeśli zachoruje zostanie przewieziony do szpitala skąd łatwiej będzie go odbić. Czy matka zdecyduje się zarazić syna? Karol odwiedza Wandę przyznaje się, że ją oszukiwał, ale ostrzega też, że grozi jej aresztowanie i nalega, żeby wyjechała z Warszawy. Czy dziewczyna zdecyduje się znów oddalić od ukochanego? Rozpoczyna się akcja odbicia Władka.',
         },
     }, {
         'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
+        'md5': 'c3b15ed1af288131115ff17a17c19dda',
         'info_dict': {
             'id': '17916176',
             'ext': 'mp4',
             'title': 'TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': 'true',
-        },
     }, {
         'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
+        'md5': 'c3b15ed1af288131115ff17a17c19dda',
         'info_dict': {
             'id': '17834272',
             'ext': 'mp4',
             'title': 'Na sygnale, odc. 39',
-            'description': 'Ekipa Wiktora ratuje młodą matkę, która spadła ze schodów trzymając na rękach noworodka. Okazuje się, że dziewczyna jest surogatką, a biologiczni rodzice dziecka próbują zmusić ją do oddania synka…',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': 'true',
         },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(
             'http://www.tvp.pl/sess/tvplayer.php?object_id=%s' % video_id, video_id)
 
-        title = self._og_search_title(webpage)
-        series = self._search_regex(
-            r'{name:\s*([\'"])SeriesTitle\1,\s*value:\s*\1(?P<series>.*?)\1},',
+        title = self._search_regex(
+            r'name\s*:\s*([\'"])Title\1\s*,\s*value\s*:\s*\1(?P<title>.+?)\1',
+            webpage, 'title', group='title')
+        series_title = self._search_regex(
+            r'name\s*:\s*([\'"])SeriesTitle\1\s*,\s*value\s*:\s*\1(?P<series>.+?)\1',
             webpage, 'series', group='series', default=None)
-        if series is not None and series not in title:
-            title = '%s, %s' % (series, title)
-        description = self._og_search_description(webpage, default=None)
+        if series_title:
+            title = '%s, %s' % (series_title, title)
+
+        thumbnail = self._search_regex(
+            r"poster\s*:\s*'([^']+)'", webpage, 'thumbnail', default=None)
 
         video_url = self._search_regex(
             r'0:{src:([\'"])(?P<url>.*?)\1', webpage, 'formats', group='url', default=None)
-        if video_url is None:
+        if not video_url:
             video_url = self._download_json(
                 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id,
                 video_id)['video_url']
@@ -89,8 +87,7 @@ class TvpIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': description,
+            'thumbnail': thumbnail,
             'formats': formats,
         }
 

From 2df54b4ba81a2615d8f5a9016061eed389ba5b6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Jan 2015 21:23:51 +0600
Subject: [PATCH 549/974] [nbcnews] Ignore HTTP errors while coping with
 playlists (Closes #4749)

---
 youtube_dl/extractor/nbc.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 690c46b6a..7ae66616a 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -6,6 +6,7 @@ import json
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
+    compat_HTTPError,
 )
 from ..utils import (
     ExtractorError,
@@ -78,6 +79,16 @@ class NBCNewsIE(InfoExtractor):
             },
             'add_ie': ['ThePlatform'],
         },
+        {
+            'url': 'http://www.nbcnews.com/feature/dateline-full-episodes/full-episode-family-business-n285156',
+            'md5': 'fdbf39ab73a72df5896b6234ff98518a',
+            'info_dict': {
+                'id': 'Wjf9EDR3A_60',
+                'ext': 'mp4',
+                'title': 'FULL EPISODE: Family Business',
+                'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -115,10 +126,19 @@ class NBCNewsIE(InfoExtractor):
                 if not base_url:
                     continue
                 playlist_url = base_url + '?form=MPXNBCNewsAPI'
-                all_videos = self._download_json(playlist_url, title)['videos']
 
                 try:
-                    info = next(v for v in all_videos if v['mpxId'] == mpxid)
+                    all_videos = self._download_json(playlist_url, title)
+                except ExtractorError as ee:
+                    if isinstance(ee.cause, compat_HTTPError):
+                        continue
+                    raise
+
+                if not all_videos or not 'videos' in all_videos:
+                    continue
+
+                try:
+                    info = next(v for v in all_videos['videos'] if v['mpxId'] == mpxid)
                     break
                 except StopIteration:
                     continue

From 564bb5e964fc263012784baa579a10eee822a436 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 02:15:28 +0600
Subject: [PATCH 550/974] [tinypic] Tweak VALID_URL regex (Closes #4754)

---
 youtube_dl/extractor/tinypic.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tinypic.py b/youtube_dl/extractor/tinypic.py
index 4fe89dbe5..e036b8cdf 100644
--- a/youtube_dl/extractor/tinypic.py
+++ b/youtube_dl/extractor/tinypic.py
@@ -9,17 +9,23 @@ from ..utils import ExtractorError
 class TinyPicIE(InfoExtractor):
     IE_NAME = 'tinypic'
     IE_DESC = 'tinypic.com videos'
-    _VALID_URL = r'http://tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'
+    _VALID_URL = r'http://(?:.+?\.)?tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'
 
-    _TEST = {
-        'url': 'http://tinypic.com/player.php?v=6xw7tc%3E&s=5#.UtqZmbRFCM8',
-        'md5': '609b74432465364e72727ebc6203f044',
-        'info_dict': {
-            'id': '6xw7tc',
-            'ext': 'flv',
-            'title': 'shadow phenomenon weird',
+    _TESTS = [
+        {
+            'url': 'http://tinypic.com/player.php?v=6xw7tc%3E&s=5#.UtqZmbRFCM8',
+            'md5': '609b74432465364e72727ebc6203f044',
+            'info_dict': {
+                'id': '6xw7tc',
+                'ext': 'flv',
+                'title': 'shadow phenomenon weird',
+            },
+        },
+        {
+            'url': 'http://de.tinypic.com/player.php?v=dy90yh&s=8',
+            'only_matching': True,
         }
-    }
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 5fe511258998666e7617c22a5db27506c16eb955 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Jan 2015 10:27:18 +0100
Subject: [PATCH 551/974] [CNNArticle] Update test

---
 youtube_dl/extractor/cnn.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 93e8d0de3..4bb58f25e 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -143,13 +143,13 @@ class CNNArticleIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:edition|www)\.)?cnn\.com/(?!video/)'
     _TEST = {
         'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
-        'md5': '275b326f85d80dff7592a9820f5dc887',
+        'md5': '689034c2a3d9c6dc4aa72d65a81efd01',
         'info_dict': {
-            'id': 'bestoftv/2014/12/21/sotu-crowley-president-obama-north-korea-not-going-to-be-intimidated.cnn',
+            'id': 'bestoftv/2014/12/21/ip-north-korea-obama.cnn',
             'ext': 'mp4',
-            'title': 'Obama: We\'re not going to be intimidated',
-            'description': 'md5:e735586f3dc936075fa654a4d91b21f9',
-            'upload_date': '20141220',
+            'title': 'Obama: Cyberattack not an act of war',
+            'description': 'md5:51ce6750450603795cad0cdfbd7d05c5',
+            'upload_date': '20141221',
         },
         'add_ie': ['CNN'],
     }

From efcddaebe9302eaaea1cbb5570c5c2bd25abf1fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Jan 2015 10:31:57 +0100
Subject: [PATCH 552/974] [cnn] Use edition.cnn.com for getting the information
 (fixes #4757)

Some videos (like http://edition.cnn.com/videos/us/2015/01/20/orig-yellowstone-oil-spill.cnn) will fail if we use cnn.com.
---
 youtube_dl/extractor/cnn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 4bb58f25e..90ea07438 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -51,7 +51,7 @@ class CNNIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         path = mobj.group('path')
         page_title = mobj.group('title')
-        info_url = 'http://cnn.com/video/data/3.0/%s/index.xml' % path
+        info_url = 'http://edition.cnn.com/video/data/3.0/%s/index.xml' % path
         info = self._download_xml(info_url, page_title)
 
         formats = []

From 47e0e1e0e2306dd7119844bbbc3ef39ccb175ed1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Jan 2015 10:36:15 +0100
Subject: [PATCH 553/974] [nbc] Fix pep8 issue

---
 youtube_dl/extractor/nbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 7ae66616a..f840f6532 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -134,7 +134,7 @@ class NBCNewsIE(InfoExtractor):
                         continue
                     raise
 
-                if not all_videos or not 'videos' in all_videos:
+                if not all_videos or 'videos' not in all_videos:
                     continue
 
                 try:

From c8dc41a6e7eea2a1235238cfeadb4caf5619d0ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 21 Jan 2015 23:05:47 +0800
Subject: [PATCH 554/974] [StreetVoice] Add new extractor

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/streetvoice.py | 36 +++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/streetvoice.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2b9d4455d..9e942e6f9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -409,6 +409,7 @@ from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
+from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
 from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
diff --git a/youtube_dl/extractor/streetvoice.py b/youtube_dl/extractor/streetvoice.py
new file mode 100644
index 000000000..55ceea75b
--- /dev/null
+++ b/youtube_dl/extractor/streetvoice.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StreetVoiceIE(InfoExtractor):
+    _VALID_URL = r'http://tw.streetvoice.com/[^/]+/songs/(?P<id>[0-9]+)/'
+    _TESTS = [
+        {
+            'url': 'http://tw.streetvoice.com/skippylu/songs/94440/',
+            'md5': '15974627fc01a29e492c98593c2fd472',
+            'info_dict': {
+                'id': '94440',
+                'ext': 'mp3',
+                'title': '輸',
+                'description': '輸 - Crispy脆樂團'
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+
+        api_url = 'http://tw.streetvoice.com/music/api/song/%s/' % song_id
+        info_dict = self._download_json(api_url, song_id)
+
+        author = info_dict['musician']['name']
+        title = info_dict['name']
+        return {
+            'id': song_id,
+            'ext': 'mp3',
+            'title': title,
+            'url': info_dict['file'],
+            'description': '%s - %s' % (title, author)
+        }

From c5db6bb32ba708bc5c92a68487b07be2374f416d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 22:27:21 +0600
Subject: [PATCH 555/974] [twitch] Refactor and add support for past broadcasts

---
 youtube_dl/extractor/__init__.py |   8 +-
 youtube_dl/extractor/twitch.py   | 345 ++++++++++++++++---------------
 2 files changed, 184 insertions(+), 169 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2b9d4455d..0b0f3be19 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -458,7 +458,13 @@ from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .twentyfourvideo import TwentyFourVideoIE
-from .twitch import TwitchIE
+from .twitch import (
+    TwitchVideoIE,
+    TwitchChapterIE,
+    TwitchVodIE,
+    TwitchProfileIE,
+    TwitchPastBroadcastsIE,
+)
 from .ubu import UbuIE
 from .udemy import (
     UdemyIE,
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b11a1d561..f70002f68 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -15,44 +15,11 @@ from ..utils import (
 )
 
 
-class TwitchIE(InfoExtractor):
-    # TODO: One broadcast may be split into multiple videos. The key
-    # 'broadcast_id' is the same for all parts, and 'broadcast_part'
-    # starts at 1 and increases. Can we treat all parts as one video?
-    _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?twitch\.tv/
-        (?:
-            (?P<channelid>[^/]+)|
-            (?:(?:[^/]+)/v/(?P<vodid>[^/]+))|
-            (?:(?:[^/]+)/b/(?P<videoid>[^/]+))|
-            (?:(?:[^/]+)/c/(?P<chapterid>[^/]+))
-        )
-        /?(?:\#.*)?$
-        """
-    _PAGE_LIMIT = 100
+class TwitchBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'http://(?:www\.)?twitch\.tv'
+
     _API_BASE = 'https://api.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
-    _TESTS = [{
-        'url': 'http://www.twitch.tv/riotgames/b/577357806',
-        'info_dict': {
-            'id': 'a577357806',
-            'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
-        },
-        'playlist_mincount': 12,
-    }, {
-        'url': 'http://www.twitch.tv/acracingleague/c/5285812',
-        'info_dict': {
-            'id': 'c5285812',
-            'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
-        },
-        'playlist_mincount': 3,
-    }, {
-        'url': 'http://www.twitch.tv/vanillatv',
-        'info_dict': {
-            'id': 'vanillatv',
-            'title': 'VanillaTV',
-        },
-        'playlist_mincount': 412,
-    }]
 
     def _handle_error(self, response):
         if not isinstance(response, dict):
@@ -64,71 +31,10 @@ class TwitchIE(InfoExtractor):
                 expected=True)
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata'):
-        response = super(TwitchIE, self)._download_json(url, video_id, note)
+        response = super(TwitchBaseIE, self)._download_json(url, video_id, note)
         self._handle_error(response)
         return response
 
-    def _extract_media(self, item, item_id):
-        ITEMS = {
-            'a': 'video',
-            'v': 'vod',
-            'c': 'chapter',
-        }
-        info = self._extract_info(self._download_json(
-            '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
-            'Downloading %s info JSON' % ITEMS[item]))
-
-        if item == 'v':
-            access_token = self._download_json(
-                '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
-                'Downloading %s access token' % ITEMS[item])
-            formats = self._extract_m3u8_formats(
-                'http://usher.twitch.tv/vod/%s?nauth=%s&nauthsig=%s'
-                % (item_id, access_token['token'], access_token['sig']),
-                item_id, 'mp4')
-            info['formats'] = formats
-            return info
-
-        response = self._download_json(
-            '%s/api/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
-            'Downloading %s playlist JSON' % ITEMS[item])
-        entries = []
-        chunks = response['chunks']
-        qualities = list(chunks.keys())
-        for num, fragment in enumerate(zip(*chunks.values()), start=1):
-            formats = []
-            for fmt_num, fragment_fmt in enumerate(fragment):
-                format_id = qualities[fmt_num]
-                fmt = {
-                    'url': fragment_fmt['url'],
-                    'format_id': format_id,
-                    'quality': 1 if format_id == 'live' else 0,
-                }
-                m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
-                if m:
-                    fmt['height'] = int(m.group('height'))
-                formats.append(fmt)
-            self._sort_formats(formats)
-            entry = dict(info)
-            entry['id'] = '%s_%d' % (entry['id'], num)
-            entry['title'] = '%s part %d' % (entry['title'], num)
-            entry['formats'] = formats
-            entries.append(entry)
-        return self.playlist_result(entries, info['id'], info['title'])
-
-    def _extract_info(self, info):
-        return {
-            'id': info['_id'],
-            'title': info['title'],
-            'description': info['description'],
-            'duration': info['length'],
-            'thumbnail': info['preview'],
-            'uploader': info['channel']['display_name'],
-            'uploader_id': info['channel']['name'],
-            'timestamp': parse_iso8601(info['recorded_at']),
-            'view_count': info['views'],
-        }
-
     def _real_initialize(self):
         self._login()
 
@@ -167,81 +73,184 @@ class TwitchIE(InfoExtractor):
             raise ExtractorError(
                 'Unable to login: %s' % m.group('msg').strip(), expected=True)
 
+
+class TwitchItemBaseIE(TwitchBaseIE):
+    def _download_info(self, item, item_id):
+        return self._extract_info(self._download_json(
+            '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
+            'Downloading %s info JSON' % self._ITEM_TYPE))
+
+    def _extract_media(self, item_id):
+        info = self._download_info(self._ITEM_SHORTCUT, item_id)
+        response = self._download_json(
+            '%s/api/videos/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
+            'Downloading %s playlist JSON' % self._ITEM_TYPE)
+        entries = []
+        chunks = response['chunks']
+        qualities = list(chunks.keys())
+        for num, fragment in enumerate(zip(*chunks.values()), start=1):
+            formats = []
+            for fmt_num, fragment_fmt in enumerate(fragment):
+                format_id = qualities[fmt_num]
+                fmt = {
+                    'url': fragment_fmt['url'],
+                    'format_id': format_id,
+                    'quality': 1 if format_id == 'live' else 0,
+                }
+                m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
+                if m:
+                    fmt['height'] = int(m.group('height'))
+                formats.append(fmt)
+            self._sort_formats(formats)
+            entry = dict(info)
+            entry['id'] = '%s_%d' % (entry['id'], num)
+            entry['title'] = '%s part %d' % (entry['title'], num)
+            entry['formats'] = formats
+            entries.append(entry)
+        return self.playlist_result(entries, info['id'], info['title'])
+
+    def _extract_info(self, info):
+        return {
+            'id': info['_id'],
+            'title': info['title'],
+            'description': info['description'],
+            'duration': info['length'],
+            'thumbnail': info['preview'],
+            'uploader': info['channel']['display_name'],
+            'uploader_id': info['channel']['name'],
+            'timestamp': parse_iso8601(info['recorded_at']),
+            'view_count': info['views'],
+        }
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if mobj.group('chapterid'):
-            return self._extract_media('c', mobj.group('chapterid'))
+        return self._extract_media(self._match_id(url))
 
-            """
-            webpage = self._download_webpage(url, chapter_id)
-            m = re.search(r'PP\.archive_id = "([0-9]+)";', webpage)
-            if not m:
-                raise ExtractorError('Cannot find archive of a chapter')
-            archive_id = m.group(1)
 
-            api = api_base + '/broadcast/by_chapter/%s.xml' % chapter_id
-            doc = self._download_xml(
-                api, chapter_id,
-                note='Downloading chapter information',
-                errnote='Chapter information download failed')
-            for a in doc.findall('.//archive'):
-                if archive_id == a.find('./id').text:
-                    break
-            else:
-                raise ExtractorError('Could not find chapter in chapter information')
+class TwitchVideoIE(TwitchItemBaseIE):
+    IE_NAME = 'twitch:video'
+    _VALID_URL = r'%s/[^/]+/b/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _ITEM_TYPE = 'video'
+    _ITEM_SHORTCUT = 'a'
 
-            video_url = a.find('./video_file_url').text
-            video_ext = video_url.rpartition('.')[2] or 'flv'
+    _TEST = {
+        'url': 'http://www.twitch.tv/riotgames/b/577357806',
+        'info_dict': {
+            'id': 'a577357806',
+            'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
+        },
+        'playlist_mincount': 12,
+    }
 
-            chapter_api_url = 'https://api.twitch.tv/kraken/videos/c' + chapter_id
-            chapter_info = self._download_json(
-                chapter_api_url, 'c' + chapter_id,
-                note='Downloading chapter metadata',
-                errnote='Download of chapter metadata failed')
 
-            bracket_start = int(doc.find('.//bracket_start').text)
-            bracket_end = int(doc.find('.//bracket_end').text)
+class TwitchChapterIE(TwitchItemBaseIE):
+    IE_NAME = 'twitch:chapter'
+    _VALID_URL = r'%s/[^/]+/c/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _ITEM_TYPE = 'chapter'
+    _ITEM_SHORTCUT = 'c'
 
-            # TODO determine start (and probably fix up file)
-            #  youtube-dl -v http://www.twitch.tv/firmbelief/c/1757457
-            #video_url += '?start=' + TODO:start_timestamp
-            # bracket_start is 13290, but we want 51670615
-            self._downloader.report_warning('Chapter detected, but we can just download the whole file. '
-                                            'Chapter starts at %s and ends at %s' % (formatSeconds(bracket_start), formatSeconds(bracket_end)))
+    _TEST = {
+        'url': 'http://www.twitch.tv/acracingleague/c/5285812',
+        'info_dict': {
+            'id': 'c5285812',
+            'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
+        },
+        'playlist_mincount': 3,
+    }
 
-            info = {
-                'id': 'c' + chapter_id,
-                'url': video_url,
-                'ext': video_ext,
-                'title': chapter_info['title'],
-                'thumbnail': chapter_info['preview'],
-                'description': chapter_info['description'],
-                'uploader': chapter_info['channel']['display_name'],
-                'uploader_id': chapter_info['channel']['name'],
-            }
-            return info
-            """
-        elif mobj.group('videoid'):
-            return self._extract_media('a', mobj.group('videoid'))
-        elif mobj.group('vodid'):
-            return self._extract_media('v', mobj.group('vodid'))
-        elif mobj.group('channelid'):
-            channel_id = mobj.group('channelid')
-            info = self._download_json(
-                '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
-                channel_id, 'Downloading channel info JSON')
-            channel_name = info.get('display_name') or info.get('name')
-            entries = []
-            offset = 0
-            limit = self._PAGE_LIMIT
-            for counter in itertools.count(1):
-                response = self._download_json(
-                    '%s/kraken/channels/%s/videos/?offset=%d&limit=%d'
-                    % (self._API_BASE, channel_id, offset, limit),
-                    channel_id, 'Downloading channel videos JSON page %d' % counter)
-                videos = response['videos']
-                if not videos:
-                    break
-                entries.extend([self.url_result(video['url'], 'Twitch') for video in videos])
-                offset += limit
-            return self.playlist_result(entries, channel_id, channel_name)
+
+class TwitchVodIE(TwitchItemBaseIE):
+    IE_NAME = 'twitch:vod'
+    _VALID_URL = r'%s/[^/]+/v/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _ITEM_TYPE = 'vod'
+    _ITEM_SHORTCUT = 'v'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/ksptv/v/3622000',
+        'info_dict': {
+            'id': 'v3622000',
+            'ext': 'mp4',
+            'title': '''KSPTV: Squadcast: "Everyone's on vacation so here's Dahud" Edition!''',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 6951,
+            'timestamp': 1419028564,
+            'upload_date': '20141219',
+            'uploader': 'KSPTV',
+            'uploader_id': 'ksptv',
+            'view_count': int,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        item_id = self._match_id(url)
+        info = self._download_info(self._ITEM_SHORTCUT, item_id)
+        access_token = self._download_json(
+            '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
+            'Downloading %s access token' % self._ITEM_TYPE)
+        formats = self._extract_m3u8_formats(
+            'http://usher.twitch.tv/vod/%s?nauth=%s&nauthsig=%s'
+            % (item_id, access_token['token'], access_token['sig']),
+            item_id, 'mp4')
+        info['formats'] = formats
+        return info
+
+
+class TwitchPlaylistBaseIE(TwitchBaseIE):
+    _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
+    _PAGE_LIMIT = 100
+
+    def _extract_playlist(self, channel_id):
+        info = self._download_json(
+            '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
+            channel_id, 'Downloading channel info JSON')
+        channel_name = info.get('display_name') or info.get('name')
+        entries = []
+        offset = 0
+        limit = self._PAGE_LIMIT
+        for counter in itertools.count(1):
+            response = self._download_json(
+                self._PLAYLIST_URL % (channel_id, offset, limit),
+                channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
+            videos = response['videos']
+            if not videos:
+                break
+            entries.extend([self.url_result(video['url']) for video in videos])
+            offset += limit
+        return self.playlist_result(entries, channel_id, channel_name)
+
+    def _real_extract(self, url):
+        return self._extract_playlist(self._match_id(url))
+
+
+class TwitchProfileIE(TwitchPlaylistBaseIE):
+    IE_NAME = 'twitch:profile'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _PLAYLIST_TYPE = 'profile'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/vanillatv/profile',
+        'info_dict': {
+            'id': 'vanillatv',
+            'title': 'VanillaTV',
+        },
+        'playlist_mincount': 412,
+    }
+
+
+class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
+    IE_NAME = 'twitch:profile'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
+    _PLAYLIST_TYPE = 'past broadcasts'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
+        'info_dict': {
+            'id': 'spamfish',
+            'title': 'Spamfish',
+        },
+        'playlist_mincount': 54,
+    }

From bb0aa4cb3ca1b21084e6e6229e4a69014e61ccff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 22:53:51 +0600
Subject: [PATCH 556/974] [streetvoice] Improve

---
 youtube_dl/extractor/streetvoice.py | 53 ++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/streetvoice.py b/youtube_dl/extractor/streetvoice.py
index 55ceea75b..6a57fa60a 100644
--- a/youtube_dl/extractor/streetvoice.py
+++ b/youtube_dl/extractor/streetvoice.py
@@ -2,35 +2,50 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import unified_strdate
 
 
 class StreetVoiceIE(InfoExtractor):
-    _VALID_URL = r'http://tw.streetvoice.com/[^/]+/songs/(?P<id>[0-9]+)/'
-    _TESTS = [
-        {
-            'url': 'http://tw.streetvoice.com/skippylu/songs/94440/',
-            'md5': '15974627fc01a29e492c98593c2fd472',
-            'info_dict': {
-                'id': '94440',
-                'ext': 'mp3',
-                'title': '輸',
-                'description': '輸 - Crispy脆樂團'
-            }
+    _VALID_URL = r'https?://(?:.+?\.)?streetvoice\.com/[^/]+/songs/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://streetvoice.com/skippylu/songs/94440/',
+        'md5': '15974627fc01a29e492c98593c2fd472',
+        'info_dict': {
+            'id': '94440',
+            'ext': 'mp3',
+            'filesize': 4167053,
+            'title': '輸',
+            'description': 'Crispy脆樂團 - 輸',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 260,
+            'upload_date': '20091018',
+            'uploader': 'Crispy脆樂團',
+            'uploader_id': '627810',
         }
-    ]
+    }, {
+        'url': 'http://tw.streetvoice.com/skippylu/songs/94440/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         song_id = self._match_id(url)
 
-        api_url = 'http://tw.streetvoice.com/music/api/song/%s/' % song_id
-        info_dict = self._download_json(api_url, song_id)
+        song = self._download_json(
+            'http://streetvoice.com/music/api/song/%s' % song_id, song_id)
+
+        title = song['name']
+        author = song['musician']['name']
 
-        author = info_dict['musician']['name']
-        title = info_dict['name']
         return {
             'id': song_id,
-            'ext': 'mp3',
+            'url': song['file'],
+            'filesize': song.get('size'),
             'title': title,
-            'url': info_dict['file'],
-            'description': '%s - %s' % (title, author)
+            'description': '%s - %s' % (author, title),
+            'thumbnail': self._proto_relative_url(song.get('image'), 'http:'),
+            'duration': song.get('length'),
+            'upload_date': unified_strdate(song.get('created_at')),
+            'uploader': author,
+            'uploader_id': compat_str(song['musician']['id']),
         }

From c62b44976545401af60fd35f0188fd09a93e449b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 22:56:28 +0600
Subject: [PATCH 557/974] Credit @yan12125 for streetvoice (#4758)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 35e256d49..b8bf3cb6f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -103,3 +103,4 @@ Christopher Krooss
 Ondřej Caletka
 Dinesh S
 Johan K. Jensen
+Yen Chi Hsuan

From 8bb1bdfae91246e1306ee31674c06a2ef7e42a08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 23:06:16 +0600
Subject: [PATCH 558/974] [twitch:past_broadcasts] Fix IE_NAME

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index f70002f68..8866e5182 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -241,7 +241,7 @@ class TwitchProfileIE(TwitchPlaylistBaseIE):
 
 
 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
-    IE_NAME = 'twitch:profile'
+    IE_NAME = 'twitch:past_broadcasts'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
     _PLAYLIST_TYPE = 'past broadcasts'

From e5763a7a7e630cfd94b8993a9592cb3243890e0c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 21 Jan 2015 21:47:55 +0200
Subject: [PATCH 559/974] [hearthisat] Add new extractor (Closes #4743)

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/hearthisat.py | 92 ++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 youtube_dl/extractor/hearthisat.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c41f2de0f..be3fecd0d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -176,6 +176,7 @@ from .goshgay import GoshgayIE
 from .grooveshark import GroovesharkIE
 from .groupon import GrouponIE
 from .hark import HarkIE
+from .hearthisat import HearThisAtIE
 from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
new file mode 100644
index 000000000..c1177a9c5
--- /dev/null
+++ b/youtube_dl/extractor/hearthisat.py
@@ -0,0 +1,92 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import (
+    str_to_int,
+    urlencode_postdata,
+)
+
+
+class HearThisAtIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hearthis\.at/(?P<artist>[^/]+)/(?P<title>[A-Za-z0-9\-]+)/?$'
+    _PLAYLIST_URL = 'https://hearthis.at/playlist.php'
+    _TEST = {
+        'url': 'https://hearthis.at/moofi/dr-kreep',
+        'md5': 'd594c573227a89f4256f0b03e68c80cc',
+        'info_dict': {
+            'id': '150939',
+            'ext': 'mp3',
+            'title': 'Moofi - Dr. Kreep',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1421564134,
+            'description': 'Creepy Patch. Mutable Instruments Braids Vowel + Formant Mode.',
+            'upload_date': '20150118',
+            'comment_count': int,
+            'view_count': int,
+            'like_count': int,
+            'duration': 71,
+            'categories': ['Experimental'],
+        }
+    }
+
+    def _real_extract(self, url):
+        m = re.match(self._VALID_URL, url)
+        display_id = '{artist:s} - {title:s}'.format(**m.groupdict())
+
+        webpage = self._download_webpage(url, display_id)
+        track_id = self._search_regex(
+            r'intTrackId\s*=\s*(\d+)', webpage, 'track ID')
+
+        payload = urlencode_postdata({'tracks[]': track_id})
+        req = compat_urllib_request.Request(self._PLAYLIST_URL, payload)
+        req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
+        track = self._download_json(req, track_id, 'Downloading playlist')[0]
+        title = '{artist:s} - {title:s}'.format(**track)
+
+        categories = None
+        if track.get('category'):
+            categories = [track['category']]
+
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        meta_span = r'<span[^>]+class="%s".*?</i>([^<]+)</span>'
+        view_count = str_to_int(self._search_regex(
+            meta_span % 'plays_count', webpage, 'view count', fatal=False))
+        like_count = str_to_int(self._search_regex(
+            meta_span % 'likes_count', webpage, 'like count', fatal=False))
+        comment_count = str_to_int(self._search_regex(
+            meta_span % 'comment_count', webpage, 'comment count', fatal=False))
+        duration = str_to_int(self._search_regex(
+            r'data-length="(\d+)', webpage, 'duration', fatal=False))
+        timestamp = str_to_int(self._search_regex(
+            r'<span[^>]+class="calctime"[^>]+data-time="(\d+)', webpage, 'timestamp', fatal=False))
+
+        track_url = self._search_regex(
+            r'<a[^>]+data-mp3="([^"]+)"', webpage, 'track URL')
+
+        formats = [{
+            'format_id': 'mp3',
+            'url': track_url,
+            'vcodec': 'none',
+        }]
+
+        return {
+            'id': track_id,
+            'display-id': display_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'like_count': like_count,
+            'categories': categories,
+        }

From b55ee18ff3a9642fe25a977e1152472877294493 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 12:04:07 +0100
Subject: [PATCH 560/974] [hearthisat] Add support for more high-quality
 download links

---
 youtube_dl/extractor/hearthisat.py | 49 ++++++++++++++++++++++--------
 youtube_dl/utils.py                |  8 +++++
 2 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
index c1177a9c5..d83072c1a 100644
--- a/youtube_dl/extractor/hearthisat.py
+++ b/youtube_dl/extractor/hearthisat.py
@@ -4,10 +4,15 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
+from ..compat import (
+    compat_urllib_request,
+    compat_urlparse,
+)
 from ..utils import (
+    HEADRequest,
     str_to_int,
     urlencode_postdata,
+    urlhandle_detect_ext,
 )
 
 
@@ -16,10 +21,10 @@ class HearThisAtIE(InfoExtractor):
     _PLAYLIST_URL = 'https://hearthis.at/playlist.php'
     _TEST = {
         'url': 'https://hearthis.at/moofi/dr-kreep',
-        'md5': 'd594c573227a89f4256f0b03e68c80cc',
+        'md5': 'ab6ec33c8fed6556029337c7885eb4e0',
         'info_dict': {
             'id': '150939',
-            'ext': 'mp3',
+            'ext': 'wav',
             'title': 'Moofi - Dr. Kreep',
             'thumbnail': 're:^https?://.*\.jpg$',
             'timestamp': 1421564134,
@@ -67,18 +72,38 @@ class HearThisAtIE(InfoExtractor):
         timestamp = str_to_int(self._search_regex(
             r'<span[^>]+class="calctime"[^>]+data-time="(\d+)', webpage, 'timestamp', fatal=False))
 
-        track_url = self._search_regex(
-            r'<a[^>]+data-mp3="([^"]+)"', webpage, 'track URL')
-
-        formats = [{
-            'format_id': 'mp3',
-            'url': track_url,
-            'vcodec': 'none',
-        }]
+        formats = []
+        mp3_url = self._search_regex(
+            r'(?s)<a class="player-link"\s+(?:[a-zA-Z0-9_:-]+="[^"]+"\s+)*?data-mp3="([^"]+)"',
+            webpage, 'title', fatal=False)
+        if mp3_url:
+            formats.append({
+                'format_id': 'mp3',
+                'vcodec': 'none',
+                'acodec': 'mp3',
+                'url': mp3_url,
+            })
+        download_path = self._search_regex(
+            r'<a class="[^"]*download_fct[^"]*"\s+href="([^"]+)"',
+            webpage, 'download URL', default=None)
+        if download_path:
+            download_url = compat_urlparse.urljoin(url, download_path)
+            ext_req = HEADRequest(download_url)
+            ext_handle = self._request_webpage(
+                ext_req, display_id, note='Determining extension')
+            ext = urlhandle_detect_ext(ext_handle)
+            formats.append({
+                'format_id': 'download',
+                'vcodec': 'none',
+                'ext': ext,
+                'url': download_url,
+                'preference': 2,  # Usually better quality
+            })
+        self._sort_formats(formats)
 
         return {
             'id': track_id,
-            'display-id': display_id,
+            'display_id': display_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7832ed87f..764474c33 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1612,6 +1612,14 @@ def urlhandle_detect_ext(url_handle):
     except AttributeError:  # Python < 3
         getheader = url_handle.info().getheader
 
+    cd = getheader('Content-Disposition')
+    if cd:
+        m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
+        if m:
+            e = determine_ext(m.group('filename'), default_ext=None)
+            if e:
+                return e
+
     return getheader('Content-Type').split("/")[1]
 
 

From 898c23c03f634d14fa414f0e0693d5c4033b2671 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 12:04:26 +0100
Subject: [PATCH 561/974] release 2015.01.22

---
 README.md             | 3 ++-
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 078e9df82..60bd7e2b6 100644
--- a/README.md
+++ b/README.md
@@ -304,7 +304,8 @@ which means you can modify it, redistribute it or use it however you like.
 
 ## Authentication Options:
     -u, --username USERNAME          login with this account ID
-    -p, --password PASSWORD          account password
+    -p, --password PASSWORD          account password. If this option is left
+                                     out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        two-factor auth code
     -n, --netrc                      use .netrc authentication data
     --video-password PASSWORD        video password (vimeo, smotri)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 63a79a7ee..f5df7be31 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.16'
+__version__ = '2015.01.22'

From bb6e38787dcc3593a4d9cb66bcf9f736570e944a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 22 Jan 2015 18:35:53 +0200
Subject: [PATCH 562/974] [videomega] Fix extraction (Closes #4763)

---
 youtube_dl/extractor/videomega.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 9fc64d172..fc6e05fe0 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -1,12 +1,15 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
 from ..utils import (
+    ExtractorError,
     remove_start,
 )
 
@@ -35,8 +38,11 @@ class VideoMegaIE(InfoExtractor):
         req.add_header('Referer', url)
         webpage = self._download_webpage(req, video_id)
 
-        escaped_data = self._search_regex(
-            r'unescape\("([^"]+)"\)', webpage, 'escaped data')
+        try:
+            escaped_data = re.findall(r'unescape\("([^"]+)"\)', webpage)[-1]
+        except IndexError:
+            raise ExtractorError('Unable to extract escaped data')
+
         playlist = compat_urllib_parse.unquote(escaped_data)
 
         thumbnail = self._search_regex(

From 240b9b7a5cd5f19f50ae28fb3ef991fede216ef6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jan 2015 23:11:22 +0600
Subject: [PATCH 563/974] [twitch] Add support for streams (Closes #893, closes
 #3693, closes #1884)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/twitch.py   | 97 +++++++++++++++++++++++++++++++-
 2 files changed, 95 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index be3fecd0d..a983fff76 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -466,6 +466,7 @@ from .twitch import (
     TwitchVodIE,
     TwitchProfileIE,
     TwitchPastBroadcastsIE,
+    TwitchStreamIE,
 )
 from .ubu import UbuIE
 from .udemy import (
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8866e5182..2891c4680 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -6,6 +6,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_str,
     compat_urllib_parse,
     compat_urllib_request,
 )
@@ -16,9 +17,10 @@ from ..utils import (
 
 
 class TwitchBaseIE(InfoExtractor):
-    _VALID_URL_BASE = r'http://(?:www\.)?twitch\.tv'
+    _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
 
     _API_BASE = 'https://api.twitch.tv'
+    _USHER_BASE = 'http://usher.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
 
     def _handle_error(self, response):
@@ -191,8 +193,8 @@ class TwitchVodIE(TwitchItemBaseIE):
             '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
             'Downloading %s access token' % self._ITEM_TYPE)
         formats = self._extract_m3u8_formats(
-            'http://usher.twitch.tv/vod/%s?nauth=%s&nauthsig=%s'
-            % (item_id, access_token['token'], access_token['sig']),
+            '%s/vod/%s?nauth=%s&nauthsig=%s'
+            % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
             item_id, 'mp4')
         info['formats'] = formats
         return info
@@ -254,3 +256,92 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
         },
         'playlist_mincount': 54,
     }
+
+
+class TwitchStreamIE(TwitchBaseIE):
+    IE_NAME = 'twitch:stream'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/shroomztv',
+        'info_dict': {
+            'id': '12772022048',
+            'display_id': 'shroomztv',
+            'ext': 'mp4',
+            'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
+            'is_live': True,
+            'timestamp': 1421928037,
+            'upload_date': '20150122',
+            'uploader': 'ShroomzTV',
+            'uploader_id': 'shroomztv',
+            'view_count': int,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        stream = self._download_json(
+            '%s/kraken/streams/%s' % (self._API_BASE, channel_id), channel_id,
+            'Downloading stream JSON').get('stream')
+
+        # Fallback on profile extraction if stream is offline
+        if not stream:
+            return self.url_result(
+                'http://www.twitch.tv/%s/profile' % channel_id,
+                'TwitchProfile', channel_id)
+
+        access_token = self._download_json(
+            '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
+            'Downloading channel access token')
+
+        query = {
+            'allow_source': 'true',
+            'p': '9386337',
+            'player': 'twitchweb',
+            'segment_preference': '4',
+            'sig': access_token['sig'],
+            'token': access_token['token'],
+        }
+
+        formats = self._extract_m3u8_formats(
+            '%s/api/channel/hls/%s.m3u8?%s'
+            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
+            channel_id, 'mp4')
+
+        view_count = stream.get('viewers')
+        timestamp = parse_iso8601(stream.get('created_at'))
+
+        channel = stream['channel']
+        title = self._live_title(channel.get('display_name') or channel.get('name'))
+        description = channel.get('status')
+
+        thumbnails = []
+        for thumbnail_key, thumbnail_url in stream['preview'].items():
+            m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
+            if not m:
+                continue
+            thumbnails.append({
+                'url': thumbnail_url,
+                'width': int(m.group('width')),
+                'height': int(m.group('height')),
+            })
+
+        return {
+            'id': compat_str(stream['_id']),
+            'display_id': channel_id,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'uploader': channel.get('display_name'),
+            'uploader_id': channel.get('name'),
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'formats': formats,
+            'is_live': True,
+        }
\ No newline at end of file

From 7932de6352f1fcb0418e78dc049fe60b92ca6c68 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:15:04 +0100
Subject: [PATCH 564/974] [hearthisat] Correct error message

---
 youtube_dl/extractor/hearthisat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
index d83072c1a..a19b31ac0 100644
--- a/youtube_dl/extractor/hearthisat.py
+++ b/youtube_dl/extractor/hearthisat.py
@@ -75,7 +75,7 @@ class HearThisAtIE(InfoExtractor):
         formats = []
         mp3_url = self._search_regex(
             r'(?s)<a class="player-link"\s+(?:[a-zA-Z0-9_:-]+="[^"]+"\s+)*?data-mp3="([^"]+)"',
-            webpage, 'title', fatal=False)
+            webpage, 'mp3 URL', fatal=False)
         if mp3_url:
             formats.append({
                 'format_id': 'mp3',

From 650ab5beebfa555383365602c914729249fad4c6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:15:58 +0100
Subject: [PATCH 565/974] [comedycentral:shows] Remove references to colbert
 report

---
 test/test_all_urls.py                 | 2 --
 youtube_dl/extractor/comedycentral.py | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index bd4fe17bf..88fda76a4 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -115,8 +115,6 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch(':ythistory', ['youtube:history'])
         self.assertMatch(':thedailyshow', ['ComedyCentralShows'])
         self.assertMatch(':tds', ['ComedyCentralShows'])
-        self.assertMatch(':colbertreport', ['ComedyCentralShows'])
-        self.assertMatch(':cr', ['ComedyCentralShows'])
 
     def test_vimeo_matching(self):
         self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel'])
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 8d27af5e5..15ca361f0 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -34,12 +34,12 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
 
 class ComedyCentralShowsIE(MTVServicesInfoExtractor):
     IE_DESC = 'The Daily Show / The Colbert Report'
-    # urls can be abbreviations like :thedailyshow or :colbert
+    # urls can be abbreviations like :thedailyshow
     # urls for episodes like:
     # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day
     #                     or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news
     #                     or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524
-    _VALID_URL = r'''(?x)^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport)
+    _VALID_URL = r'''(?x)^(:(?P<shortname>tds|thedailyshow)
                       |https?://(:www\.)?
                           (?P<showname>thedailyshow|thecolbertreport)\.(?:cc\.)?com/
                          ((?:full-)?episodes/(?:[0-9a-z]{6}/)?(?P<episode>.*)|

From 78111136db788918452a80233e74fd4c506eb6f8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:18:21 +0100
Subject: [PATCH 566/974] [twitch] Move URL matching tests into extractor

---
 test/test_all_urls.py          | 13 -------------
 youtube_dl/extractor/twitch.py |  7 +++++--
 2 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index 88fda76a4..e66264b4b 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -14,7 +14,6 @@ from test.helper import gettestcases
 from youtube_dl.extractor import (
     FacebookIE,
     gen_extractors,
-    TwitchIE,
     YoutubeIE,
 )
 
@@ -72,18 +71,6 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
         self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
 
-    def test_twitch_channelid_matching(self):
-        self.assertTrue(TwitchIE.suitable('twitch.tv/vanillatv'))
-        self.assertTrue(TwitchIE.suitable('www.twitch.tv/vanillatv'))
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv'))
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/'))
-
-    def test_twitch_videoid_matching(self):
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/b/328087483'))
-
-    def test_twitch_chapterid_matching(self):
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361'))
-
     def test_youtube_extract(self):
         assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
         assertExtractId('http://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8866e5182..017eff742 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -148,14 +148,17 @@ class TwitchChapterIE(TwitchItemBaseIE):
     _ITEM_TYPE = 'chapter'
     _ITEM_SHORTCUT = 'c'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.twitch.tv/acracingleague/c/5285812',
         'info_dict': {
             'id': 'c5285812',
             'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
         },
         'playlist_mincount': 3,
-    }
+    }, {
+        'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
+        'only_matching': True,
+    }]
 
 
 class TwitchVodIE(TwitchItemBaseIE):

From b4a64c592b5f0f547c11c72e6c840612b336de5d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:21:17 +0100
Subject: [PATCH 567/974] [README] Add an FAQ entry about destination folder

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 60bd7e2b6..efa48f280 100644
--- a/README.md
+++ b/README.md
@@ -488,6 +488,10 @@ To make a different directory work - either for ffmpeg, or for youtube-dl, or fo
 
 From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
 
+### How do I put downloads into a specific folder?
+
+Use the `-o` to specify an [output template](#output-template), for example `-o "/home/user/videos/%(title)s-%(id)s.%(ext)s"`. If you want this for all of your downloads, put the option into your [configuration file](#configuration).
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.

From f353cbdb2f91564d4637462875c1a12bfbcd2ada Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jan 2015 23:34:40 +0600
Subject: [PATCH 568/974] [twitch:stream] Randomize query

---
 youtube_dl/extractor/twitch.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 14314916a..87e3c453d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import itertools
 import re
+import random
 
 from .common import InfoExtractor
 from ..compat import (
@@ -305,7 +306,7 @@ class TwitchStreamIE(TwitchBaseIE):
 
         query = {
             'allow_source': 'true',
-            'p': '9386337',
+            'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',
             'sig': access_token['sig'],

From dc1b027cd4672c6b4772c12daf2a6a2a93469270 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 23:06:03 +0100
Subject: [PATCH 569/974] [twitch] PEP8

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 14314916a..6831bb596 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -347,4 +347,4 @@ class TwitchStreamIE(TwitchBaseIE):
             'view_count': view_count,
             'formats': formats,
             'is_live': True,
-        }
\ No newline at end of file
+        }

From 50789175edbe3aaad77f45e6fe883ba09580cc6f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 23:48:58 +0100
Subject: [PATCH 570/974] [pornhub] Detect private videos and emit an error
 message (Closes #4764)

---
 youtube_dl/extractor/pornhub.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 634142d0d..fb2032832 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -10,6 +10,7 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
+    ExtractorError,
     str_to_int,
 )
 from ..aes import (
@@ -44,6 +45,15 @@ class PornHubIE(InfoExtractor):
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
+        error_msg = self._html_search_regex(
+            r'(?s)<div class="userMessageSection[^"]*".*?>(.*?)</div>',
+            webpage, 'error message', default=None)
+        if error_msg:
+            error_msg = re.sub(r'\s+', ' ', error_msg)
+            raise ExtractorError(
+                'PornHub said: %s' % error_msg,
+                expected=True, video_id=video_id)
+
         video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
         video_uploader = self._html_search_regex(
             r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|<span class="username)[^>]+>(.+?)<',

From 083c9df93b08a24e967b68fbdd2f4a71ae74c8c8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:04:05 +0100
Subject: [PATCH 571/974] [YoutubeDL] Allow filtering by properties (Fixes
 #4584)

---
 test/test_YoutubeDL.py  | 55 +++++++++++++++++++++++++++++++++++++++++
 youtube_dl/YoutubeDL.py | 54 ++++++++++++++++++++++++++++++++++++++++
 youtube_dl/options.py   | 11 +++++++++
 3 files changed, 120 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 85d87f2c3..678b9f7d1 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -281,6 +281,61 @@ class TestFormatSelection(unittest.TestCase):
             downloaded = ydl.downloaded_info_dicts[0]
             self.assertEqual(downloaded['format_id'], f1id)
 
+    def test_format_filtering(self):
+        formats = [
+            {'format_id': 'A', 'filesize': 500, 'width': 1000},
+            {'format_id': 'B', 'filesize': 1000, 'width': 500},
+            {'format_id': 'C', 'filesize': 1000, 'width': 400},
+            {'format_id': 'D', 'filesize': 2000, 'width': 600},
+            {'format_id': 'E', 'filesize': 3000},
+            {'format_id': 'F'},
+            {'format_id': 'G', 'filesize': 1000000},
+        ]
+        for f in formats:
+            f['url'] = 'http://_/'
+            f['ext'] = 'unknown'
+        info_dict = _make_result(formats)
+
+        ydl = YDL({'format': 'best[filesize<3000]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'D')
+
+        ydl = YDL({'format': 'best[filesize<=3000]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'E')
+
+        ydl = YDL({'format': 'best[filesize <= ? 3000]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'F')
+
+        ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'B')
+
+        ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'C')
+
+        ydl = YDL({'format': '[filesize>?1]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'G')
+
+        ydl = YDL({'format': '[filesize<1M]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'E')
+
+        ydl = YDL({'format': '[filesize<1MiB]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'G')
+
     def test_add_extra_info(self):
         test_dict = {
             'extractor': 'Foo',
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 772fddd45..8ef74e414 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -10,6 +10,7 @@ import io
 import itertools
 import json
 import locale
+import operator
 import os
 import platform
 import re
@@ -49,6 +50,7 @@ from .utils import (
     make_HTTPS_handler,
     MaxDownloadsReached,
     PagedList,
+    parse_filesize,
     PostProcessingError,
     platform_name,
     preferredencoding,
@@ -768,7 +770,59 @@ class YoutubeDL(object):
         else:
             raise Exception('Invalid result type: %s' % result_type)
 
+    def _apply_format_filter(self, format_spec, available_formats):
+        " Returns a tuple of the remaining format_spec and filtered formats "
+
+        OPERATORS = {
+            '<': operator.lt,
+            '<=': operator.le,
+            '>': operator.gt,
+            '>=': operator.ge,
+            '=': operator.eq,
+            '!=': operator.ne,
+        }
+        operator_rex = re.compile(r'''(?x)\s*\[
+            (?P<key>width|height|tbr|abr|vbr|filesize)
+            \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
+            (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
+            \]$
+            ''' % '|'.join(map(re.escape, OPERATORS.keys())))
+        m = operator_rex.search(format_spec)
+        if not m:
+            raise ValueError('Invalid format specification %r' % format_spec)
+
+        try:
+            comparison_value = int(m.group('value'))
+        except ValueError:
+            comparison_value = parse_filesize(m.group('value'))
+            if comparison_value is None:
+                comparison_value = parse_filesize(m.group('value') + 'B')
+            if comparison_value is None:
+                raise ValueError(
+                    'Invalid value %r in format specification %r' % (
+                        m.group('value'), format_spec))
+        op = OPERATORS[m.group('op')]
+
+        def _filter(f):
+            actual_value = f.get(m.group('key'))
+            if actual_value is None:
+                return m.group('none_inclusive')
+            return op(actual_value, comparison_value)
+        new_formats = [f for f in available_formats if _filter(f)]
+
+        new_format_spec = format_spec[:-len(m.group(0))]
+        if not new_format_spec:
+            new_format_spec = 'best'
+
+        return (new_format_spec, new_formats)
+
     def select_format(self, format_spec, available_formats):
+        while format_spec.endswith(']'):
+            format_spec, available_formats = self._apply_format_filter(
+                format_spec, available_formats)
+        if not available_formats:
+            return None
+
         if format_spec == 'best' or format_spec is None:
             return available_formats[-1]
         elif format_spec == 'worst':
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f25c12e52..fd7b400b2 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -289,6 +289,17 @@ def parseOpts(overrideArguments=None):
             'extensions aac, m4a, mp3, mp4, ogg, wav, webm. '
             'You can also use the special names "best",'
             ' "bestvideo", "bestaudio", "worst". '
+            ' You can filter the video results by putting a condition in'
+            ' brackets, as in -f "best[height=720]"'
+            ' (or -f "[filesize>10M]"). '
+            ' This works for filesize, height, width, tbr, abr, and vbr'
+            ' and the comparisons <, <=, >, >=, =, != .'
+            ' Formats for which the value is not known are excluded unless you'
+            ' put a question mark (?) after the operator.'
+            ' You can combine format filters, so  '
+            '-f "[height <=? 720][tbr>500]" '
+            'selects up to 720p videos (or videos where the height is not '
+            'known) with a bitrate of at least 500 KBit/s.'
             ' By default, youtube-dl will pick the best quality.'
             ' Use commas to download multiple audio formats, such as'
             ' -f  136/137/mp4/bestvideo,140/m4a/bestaudio.'

From c994e6bd63feb95e1e385af2333ae2f615314ac5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:06:11 +0100
Subject: [PATCH 572/974] release 2015.01.23

---
 README.md             | 20 ++++++++++++++++----
 youtube_dl/version.py |  2 +-
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index efa48f280..1408ebba0 100644
--- a/README.md
+++ b/README.md
@@ -267,10 +267,22 @@ which means you can modify it, redistribute it or use it however you like.
                                      by extension for the extensions aac, m4a,
                                      mp3, mp4, ogg, wav, webm. You can also use
                                      the special names "best", "bestvideo",
-                                     "bestaudio", "worst".  By default, youtube-
-                                     dl will pick the best quality. Use commas
-                                     to download multiple audio formats, such as
-                                     -f
+                                     "bestaudio", "worst".  You can filter the
+                                     video results by putting a condition in
+                                     brackets, as in -f "best[height=720]" (or
+                                     -f "[filesize>10M]").  This works for
+                                     filesize, height, width, tbr, abr, and vbr
+                                     and the comparisons <, <=, >, >=, =, != .
+                                     Formats for which the value is not known
+                                     are excluded unless you put a question mark
+                                     (?) after the operator. You can combine
+                                     format filters, so  -f "[height <=?
+                                     720][tbr>500]" selects up to 720p videos
+                                     (or videos where the height is not known)
+                                     with a bitrate of at least 500 KBit/s. By
+                                     default, youtube-dl will pick the best
+                                     quality. Use commas to download multiple
+                                     audio formats, such as -f
                                      136/137/mp4/bestvideo,140/m4a/bestaudio.
                                      You can merge the video and audio of two
                                      formats into a single file using -f <video-
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f5df7be31..6c552064b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.22'
+__version__ = '2015.01.23'

From c2a30b250cdea5a8c73225e6a14707a669640de0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:32:35 +0100
Subject: [PATCH 573/974] [testtube] Add new extractor (Fixes #4759)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/testtube.py | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/testtube.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a983fff76..9ab90ac62 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -432,6 +432,7 @@ from .telemb import TeleMBIE
 from .teletask import TeleTaskIE
 from .tenplay import TenPlayIE
 from .testurl import TestURLIE
+from .testtube import TestTubeIE
 from .tf1 import TF1IE
 from .theonion import TheOnionIE
 from .theplatform import ThePlatformIE
diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
new file mode 100644
index 000000000..fd47e71a2
--- /dev/null
+++ b/youtube_dl/extractor/testtube.py
@@ -0,0 +1,60 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class TestTubeIE(InfoExtractor):
+    _VALID_URL = r'https?://testtube\.com/[^/?#]+/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial',
+        'info_dict': {
+            'id': '60163',
+            'display_id': '5-weird-ways-plants-can-eat-animals',
+            'duration': 275,
+            'ext': 'mp4',
+            'title': '5 Weird Ways Plants Can Eat Animals',
+            'description': 'Why have some plants evolved to eat meat?',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'DNews',
+            'uploader_id': 'dnews',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(
+            r"player\.loadRevision3Item\('video_id',\s*([0-9]+)\);",
+            webpage, 'video ID')
+
+        all_info = self._download_json(
+            'https://testtube.com/api/getPlaylist.json?api_key=ba9c741bce1b9d8e3defcc22193f3651b8867e62&codecs=h264,vp8,theora&video_id=%s' % video_id,
+            video_id)
+        info = all_info['items'][0]
+
+        formats = []
+        for vcodec, fdatas in info['media'].items():
+            for name, fdata in fdatas.items():
+                formats.append({
+                    'format_id': '%s-%s' % (vcodec, name),
+                    'url': fdata['url'],
+                    'vcodec': vcodec,
+                    'tbr': fdata.get('bitrate'),
+                })
+        self._sort_formats(formats)
+
+        duration = int_or_none(info.get('duration'))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': info['title'],
+            'description': info.get('summary'),
+            'thumbnail': info.get('images', {}).get('large'),
+            'uploader': info.get('show', {}).get('name'),
+            'uploader_id': info.get('show', {}).get('slug'),
+            'duration': duration,
+            'formats': formats,
+        }

From 317239b097a913824e20d436cdb8c74161564268 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:33:14 +0100
Subject: [PATCH 574/974] release 2015.01.23.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6c552064b..28458fd69 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23'
+__version__ = '2015.01.23.1'

From 61ca9a80b34b14fa0e5e19e35ee76e0086d49edf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 01:21:30 +0100
Subject: [PATCH 575/974] [generic] Add support for BOMs (Fixes #4753)

---
 test/test_utils.py              | 17 +++++++++++++++++
 youtube_dl/extractor/generic.py |  3 ++-
 youtube_dl/utils.py             | 20 ++++++++++++++++++++
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 206760d99..bdd7f268a 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -28,6 +28,7 @@ from youtube_dl.utils import (
     fix_xml_ampersands,
     InAdvancePagedList,
     intlist_to_bytes,
+    is_html,
     js_to_json,
     limit_length,
     OnDemandPagedList,
@@ -417,5 +418,21 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
         self.assertTrue(age_restricted(18, 14))
         self.assertFalse(age_restricted(18, 18))
 
+    def test_is_html(self):
+        self.assertFalse(is_html(b'\x49\x44\x43<html'))
+        self.assertTrue(is_html(b'<!DOCTYPE foo>\xaaa'))
+        self.assertTrue(is_html(  # UTF-8 with BOM
+            b'\xef\xbb\xbf<!DOCTYPE foo>\xaaa'))
+        self.assertTrue(is_html(  # UTF-16-LE
+            b'\xff\xfe<\x00h\x00t\x00m\x00l\x00>\x00\xe4\x00'
+        ))
+        self.assertTrue(is_html(  # UTF-16-BE
+            b'\xfe\xff\x00<\x00h\x00t\x00m\x00l\x00>\x00\xe4'
+        ))
+        self.assertTrue(is_html(  # UTF-32-BE
+            b'\x00\x00\xFE\xFF\x00\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4'))
+        self.assertTrue(is_html(  # UTF-32-LE
+            b'\xFF\xFE\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4\x00\x00\x00'))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7a5bf9392..b893d8149 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -17,6 +17,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     HEADRequest,
+    is_html,
     orderedSet,
     parse_xml,
     smuggle_url,
@@ -647,7 +648,7 @@ class GenericIE(InfoExtractor):
         # Maybe it's a direct link to a video?
         # Be careful not to download the whole thing!
         first_bytes = full_response.read(512)
-        if not re.match(r'^\s*<', first_bytes.decode('utf-8', 'replace')):
+        if not is_html(first_bytes):
             self._downloader.report_warning(
                 'URL could be a direct video link, returning it as such.')
             upload_date = unified_strdate(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 764474c33..b433b591b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1631,3 +1631,23 @@ def age_restricted(content_limit, age_limit):
     if content_limit is None:
         return False  # Content available for everyone
     return age_limit < content_limit
+
+
+def is_html(first_bytes):
+    """ Detect whether a file contains HTML by examining its first bytes. """
+
+    BOMS = [
+        (b'\xef\xbb\xbf', 'utf-8'),
+        (b'\x00\x00\xfe\xff', 'utf-32-be'),
+        (b'\xff\xfe\x00\x00', 'utf-32-le'),
+        (b'\xff\xfe', 'utf-16-le'),
+        (b'\xfe\xff', 'utf-16-be'),
+    ]
+    for bom, enc in BOMS:
+        if first_bytes.startswith(bom):
+            s = first_bytes[len(bom):].decode(enc, 'replace')
+            break
+    else:
+        s = first_bytes.decode('utf-8', 'replace')
+
+    return re.match(r'^\s*<', s)

From 08ff6ab07e9d08853ca19cf3b8745fac11abf0f6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 01:34:24 +0100
Subject: [PATCH 576/974] [gamestar] Modernize slightly

---
 youtube_dl/extractor/gamestar.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py
index 50f8fc7e7..7591a151e 100644
--- a/youtube_dl/extractor/gamestar.py
+++ b/youtube_dl/extractor/gamestar.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -29,9 +27,7 @@ class GameStarIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         og_title = self._og_search_title(webpage)

From d79323136fabc2cd72afc7c124e17797e32df514 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 11:15:18 +0100
Subject: [PATCH 577/974] [utils] Simplify HTTPS socket creation

We were duplicating (bad) code and doing crazy things with SSL.
Just use TLSv1 across the board, and do with one implementation of HTTPSConnection.connect.
Fixes #4696.
---
 youtube_dl/utils.py | 24 +++++-------------------
 1 file changed, 5 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3536a5bd6..4be323926 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -411,25 +411,9 @@ def make_HTTPS_handler(params, **kwargs):
             pass
 
     if sys.version_info < (3, 2):
-        import httplib
-
-        class HTTPSConnectionV3(httplib.HTTPSConnection):
-            def __init__(self, *args, **kwargs):
-                httplib.HTTPSConnection.__init__(self, *args, **kwargs)
-
-            def connect(self):
-                sock = socket.create_connection((self.host, self.port), self.timeout)
-                if getattr(self, '_tunnel_host', False):
-                    self.sock = sock
-                    self._tunnel()
-                try:
-                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_TLSv1)
-                except ssl.SSLError:
-                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
-
-        return YoutubeDLHTTPSHandler(params, https_conn_class=HTTPSConnectionV3, **kwargs)
+        return YoutubeDLHTTPSHandler(params, **kwargs)
     else:  # Python < 3.4
-        context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         context.verify_mode = (ssl.CERT_NONE
                                if opts_no_check_certificate
                                else ssl.CERT_REQUIRED)
@@ -560,7 +544,9 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
                 sock = compat_socket_create_connection(
                     (self.host, self.port), self.timeout, sa)
                 if is_https:
-                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                    self.sock = ssl.wrap_socket(
+                        sock, self.key_file, self.cert_file,
+                        ssl_version=ssl.PROTOCOL_TLSv1)
                 else:
                     self.sock = sock
             hc.connect = functools.partial(_hc_connect, hc)

From ba551681573aafedf9a0a234a9522cc06b12a4b7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 11:20:24 +0100
Subject: [PATCH 578/974] release 2015.01.23.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 28458fd69..674c2f279 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.1'
+__version__ = '2015.01.23.2'

From 501f13fbf3d1f7225f91e3e0ad008df2cd3219f1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:00:25 +0100
Subject: [PATCH 579/974] [generic] Add support for Cinerama player (Fixes
 #4752)

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b893d8149..a028c4ed4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -489,6 +489,16 @@ class GenericIE(InfoExtractor):
                 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
             }
         },
+        # Cinerama player
+        {
+            'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
+            'info_dict': {
+                'id': '730m_DandD_1901_512k',
+                'ext': 'mp4',
+                'uploader': 'www.abc.net.au',
+                'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
+            }
+        }
     ]
 
     def report_following_redirect(self, new_url):
@@ -1046,6 +1056,10 @@ class GenericIE(InfoExtractor):
                     \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
                         ["']?url["']?\s*:\s*["']([^"']+)["']
             ''', webpage))
+        if not found:
+            # Cinerama player
+            found = re.findall(
+                r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
         if not found:
             # Try to find twitter cards info
             found = filter_video(re.findall(

From 649f7966f73b215f02b9c4db5a07d8c4b7e17acb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:07:13 +0100
Subject: [PATCH 580/974] Fix --sleep-interval (#3426)

---
 youtube_dl/__init__.py | 6 ------
 youtube_dl/options.py  | 2 +-
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ea1660452..7bd7295e2 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -132,11 +132,6 @@ def _real_main(argv=None):
         if numeric_limit is None:
             parser.error('invalid rate limit specified')
         opts.ratelimit = numeric_limit
-    if opts.sleepinterval is not None:
-        try:
-            opts.sleepinterval = abs(int(opts.sleepinterval))
-        except ValueError:
-            parser.error(u'invalid sleep interval specified')
     if opts.min_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
         if numeric_limit is None:
@@ -272,7 +267,6 @@ def _real_main(argv=None):
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
         'ratelimit': opts.ratelimit,
-        'sleepinterval': opts.sleepinterval,
         'nooverwrites': opts.nooverwrites,
         'retries': opts.retries,
         'buffersize': opts.buffersize,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 12c9826f8..262c60013 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -423,7 +423,7 @@ def parseOpts(overrideArguments=None):
         help='Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
     workarounds.add_option(
         '--sleep-interval', metavar='SECONDS',
-        dest='sleep_interval',
+        dest='sleep_interval', type=float,
         help='Number of seconds to sleep before each download.')
 
     verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')

From aa42e87340e491d0b151e9dad368711fc275c7b9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:17:12 +0100
Subject: [PATCH 581/974] [utils] Catch strange Windows errors (Closes #4733)

---
 youtube_dl/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4be323926..463cc20ff 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -863,6 +863,9 @@ def _windows_write_string(s, out):
     except AttributeError:
         # If the output stream doesn't have a fileno, it's virtual
         return False
+    except io.UnsupportedOperation:
+        # Some strange Windows pseudo files?
+        return False
     if fileno not in WIN_OUTPUT_IDS:
         return False
 

From 6f58db89820fe03effb2014ee70e1ffd0b1f3017 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:17:19 +0100
Subject: [PATCH 582/974] release 2015.01.23.3

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1408ebba0..869523720 100644
--- a/README.md
+++ b/README.md
@@ -259,6 +259,8 @@ which means you can modify it, redistribute it or use it however you like.
     --bidi-workaround                Work around terminals that lack
                                      bidirectional text support. Requires bidiv
                                      or fribidi executable in PATH
+    --sleep-interval SECONDS         Number of seconds to sleep before each
+                                     download.
 
 ## Video Format Options:
     -f, --format FORMAT              video format code, specify the order of
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 674c2f279..f39b94361 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.2'
+__version__ = '2015.01.23.3'

From bf7fa94ec7202bde963a75ab903996ac575910db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 16:31:52 +0100
Subject: [PATCH 583/974] [downloader/f4m] build_fragments_list: Support videos
 with more than 1 segment

---
 youtube_dl/downloader/f4m.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index c460c167a..c68b2c303 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -177,13 +177,12 @@ def build_fragments_list(boot_info):
     """ Return a list of (segment, fragment) for each fragment in the video """
     res = []
     segment_run_table = boot_info['segments'][0]
-    # I've only found videos with one segment
-    segment_run_entry = segment_run_table['segment_run'][0]
-    n_frags = segment_run_entry[1]
     fragment_run_entry_table = boot_info['fragments'][0]['fragments']
     first_frag_number = fragment_run_entry_table[0]['first']
-    for (i, frag_number) in zip(range(1, n_frags + 1), itertools.count(first_frag_number)):
-        res.append((1, frag_number))
+    fragments_counter = itertools.count(first_frag_number)
+    for segment, fragments_count in segment_run_table['segment_run']:
+        for _ in range(fragments_count):
+            res.append((segment, next(fragments_counter)))
     return res
 
 

From 0920e5830f890580ec16cdd10bfe8def73a1a09f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 16:39:23 +0100
Subject: [PATCH 584/974] [atresplayer] Don't include f4m formats if they are
 protected by DRM (fixes #4705)

---
 youtube_dl/extractor/atresplayer.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 5db1941b3..37321ef1d 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -107,7 +107,14 @@ class AtresPlayerIE(InfoExtractor):
 
             for _, video_url in fmt_json['resultObject'].items():
                 if video_url.endswith('/Manifest'):
-                    formats.extend(self._extract_f4m_formats(video_url[:-9] + '/manifest.f4m', video_id))
+                    if 'geodeswowsmpra3player' in video_url:
+                        f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
+                        f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
+                        # this videos are protected by DRM, the f4m downloader doesn't support them
+                        continue
+                    else:
+                        f4m_url = video_url[:-9] + '/manifest.f4m'
+                    formats.extend(self._extract_f4m_formats(f4m_url, video_id))
                 else:
                     formats.append({
                         'url': video_url,

From c2e64f71d075bbc916f343916ff1e679f642a821 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jan 2015 21:58:40 +0600
Subject: [PATCH 585/974] [twitch] Add support for bookmarks

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/twitch.py   | 41 ++++++++++++++++++++++++++++----
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9ab90ac62..9e1ce5db3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -467,6 +467,7 @@ from .twitch import (
     TwitchVodIE,
     TwitchProfileIE,
     TwitchPastBroadcastsIE,
+    TwitchBookmarksIE,
     TwitchStreamIE,
 )
 from .ubu import UbuIE
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 340cadcf5..741df7cbc 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -220,12 +220,18 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
             response = self._download_json(
                 self._PLAYLIST_URL % (channel_id, offset, limit),
                 channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
-            videos = response['videos']
-            if not videos:
+            page_entries = self._extract_playlist_page(response)
+            if not page_entries:
                 break
-            entries.extend([self.url_result(video['url']) for video in videos])
+            entries.extend(page_entries)
             offset += limit
-        return self.playlist_result(entries, channel_id, channel_name)
+        return self.playlist_result(
+            [self.url_result(entry) for entry in set(entries)],
+            channel_id, channel_name)
+
+    def _extract_playlist_page(self, response):
+        videos = response.get('videos')
+        return [video['url'] for video in videos] if videos else []
 
     def _real_extract(self, url):
         return self._extract_playlist(self._match_id(url))
@@ -262,6 +268,31 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
     }
 
 
+class TwitchBookmarksIE(TwitchPlaylistBaseIE):
+    IE_NAME = 'twitch:bookmarks'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
+    _PLAYLIST_TYPE = 'bookmarks'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
+        'info_dict': {
+            'id': 'ognos',
+            'title': 'Ognos',
+        },
+        'playlist_mincount': 3,
+    }
+
+    def _extract_playlist_page(self, response):
+        entries = []
+        for bookmark in response.get('bookmarks', []):
+            video = bookmark.get('video')
+            if not video:
+                continue
+            entries.append(video['url'])
+        return entries
+
+
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
     _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
@@ -348,4 +379,4 @@ class TwitchStreamIE(TwitchBaseIE):
             'view_count': view_count,
             'formats': formats,
             'is_live': True,
-        }
+        }
\ No newline at end of file

From e793f7671c3e303e4fbe2c4cd18e80ad4b5e5417 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 17:09:26 +0100
Subject: [PATCH 586/974] [liveleak] Modernize

---
 youtube_dl/extractor/liveleak.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index b04be1e8c..e56a4d6b1 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 
 
 class LiveLeakIE(InfoExtractor):
-    _VALID_URL = r'^(?:http://)?(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<video_id>[\w_]+)(?:.*)'
+    _VALID_URL = r'https?://(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)'
     _TESTS = [{
         'url': 'http://www.liveleak.com/view?i=757_1364311680',
         'md5': '0813c2430bea7a46bf13acf3406992f4',
@@ -43,8 +43,7 @@ class LiveLeakIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_title = self._og_search_title(webpage).replace('LiveLeak.com -', '').strip()

From ebd46aed5119899826629cf751ba5abe7a65d50b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jan 2015 22:21:55 +0600
Subject: [PATCH 587/974] [atresplayer] Filter URLs and clarify android format
 ids

---
 youtube_dl/extractor/atresplayer.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 37321ef1d..8fd69b971 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -105,7 +105,9 @@ class AtresPlayerIE(InfoExtractor):
                 raise ExtractorError(
                     '%s returned error: %s' % (self.IE_NAME, result), expected=True)
 
-            for _, video_url in fmt_json['resultObject'].items():
+            for format_id, video_url in fmt_json['resultObject'].items():
+                if format_id == 'token' or not video_url.startswith('http'):
+                    continue
                 if video_url.endswith('/Manifest'):
                     if 'geodeswowsmpra3player' in video_url:
                         f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
@@ -118,7 +120,7 @@ class AtresPlayerIE(InfoExtractor):
                 else:
                     formats.append({
                         'url': video_url,
-                        'format_id': 'android',
+                        'format_id': 'android-%s' % format_id,
                         'preference': 1,
                     })
         self._sort_formats(formats)

From 26e274666d285026dfb6033cdad74cc2a108e2bc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 17:22:14 +0100
Subject: [PATCH 588/974] [liveleak] Add original videos (Fixes #4768)

---
 youtube_dl/extractor/liveleak.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index e56a4d6b1..35822067f 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -11,17 +11,17 @@ class LiveLeakIE(InfoExtractor):
     _VALID_URL = r'https?://(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)'
     _TESTS = [{
         'url': 'http://www.liveleak.com/view?i=757_1364311680',
-        'md5': '0813c2430bea7a46bf13acf3406992f4',
+        'md5': '50f79e05ba149149c1b4ea961223d5b3',
         'info_dict': {
             'id': '757_1364311680',
-            'ext': 'mp4',
+            'ext': 'flv',
             'description': 'extremely bad day for this guy..!',
             'uploader': 'ljfriel2',
             'title': 'Most unlucky car accident'
         }
     }, {
         'url': 'http://www.liveleak.com/view?i=f93_1390833151',
-        'md5': 'd3f1367d14cc3c15bf24fbfbe04b9abf',
+        'md5': 'b13a29626183c9d33944e6a04f41aafc',
         'info_dict': {
             'id': 'f93_1390833151',
             'ext': 'mp4',
@@ -80,9 +80,19 @@ class LiveLeakIE(InfoExtractor):
         sources = json.loads(sources_json)
 
         formats = [{
+            'format_id': '%s' % i,
             'format_note': s.get('label'),
             'url': s['file'],
-        } for s in sources]
+        } for i, s in enumerate(sources)]
+        for i, s in enumerate(sources):
+            orig_url = s['file'].replace('.h264_base.mp4', '')
+            if s['file'] != orig_url:
+                formats.append({
+                    'format_id': 'original-%s' % i,
+                    'format_note': s.get('label'),
+                    'url': orig_url,
+                    'preference': 1,
+                })
         self._sort_formats(formats)
 
         return {

From 0c17278843a30d4aea1f8334985e6fbae17000d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jan 2015 22:54:29 +0600
Subject: [PATCH 589/974] [atresplayer] Extract subtitles

---
 youtube_dl/extractor/atresplayer.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 8fd69b971..f42862be3 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import time
 import hmac
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
@@ -17,7 +17,7 @@ from ..utils import (
 )
 
 
-class AtresPlayerIE(InfoExtractor):
+class AtresPlayerIE(SubtitlesInfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'
     _TESTS = [
         {
@@ -143,6 +143,15 @@ class AtresPlayerIE(InfoExtractor):
         description = xpath_text(art, './description', 'description')
         thumbnail = xpath_text(episode, './media/asset/files/background', 'thumbnail')
 
+        subtitles = {}
+        subtitle = xpath_text(episode, './media/asset/files/subtitle', 'subtitle')
+        if subtitle:
+            subtitles['es'] = subtitle
+
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
         return {
             'id': video_id,
             'title': title,
@@ -150,4 +159,5 @@ class AtresPlayerIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
+            'subtitles': self.extract_subtitles(video_id, subtitles),
         }

From 62cd676c7474f696804eda653558ada94c5953a0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 18:39:12 +0100
Subject: [PATCH 590/974] [youtube] Fixup DASH m4a headers

This fixes #2288, #2506, #2607, #3681, #4741, #4767.
---
 youtube_dl/YoutubeDL.py              | 28 +++++++++++++++++++++++-----
 youtube_dl/extractor/youtube.py      |  6 +++---
 youtube_dl/postprocessor/__init__.py |  2 ++
 youtube_dl/postprocessor/ffmpeg.py   | 20 +++++++++++++++++++-
 4 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8f34b17b4..521e4055e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -73,6 +73,7 @@ from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
+    FFmpegFixupM4aPP,
     FFmpegFixupStretchedPP,
     FFmpegMergerPP,
     FFmpegPostProcessor,
@@ -213,7 +214,7 @@ class YoutubeDL(object):
                        - "never": do nothing
                        - "warn": only emit a warning
                        - "detect_or_warn": check whether we can do anything
-                                           about it, warn otherwise
+                                           about it, warn otherwise (default)
     source_address:    (Experimental) Client-side IP address to bind to.
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
@@ -1219,11 +1220,12 @@ class YoutubeDL(object):
 
             if success:
                 # Fixup content
+                fixup_policy = self.params.get('fixup')
+                if fixup_policy is None:
+                    fixup_policy = 'detect_or_warn'
+
                 stretched_ratio = info_dict.get('stretched_ratio')
                 if stretched_ratio is not None and stretched_ratio != 1:
-                    fixup_policy = self.params.get('fixup')
-                    if fixup_policy is None:
-                        fixup_policy = 'detect_or_warn'
                     if fixup_policy == 'warn':
                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
                             info_dict['id'], stretched_ratio))
@@ -1237,7 +1239,23 @@ class YoutubeDL(object):
                                 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
                                     info_dict['id'], stretched_ratio))
                     else:
-                        assert fixup_policy == 'ignore'
+                        assert fixup_policy in ('ignore', 'never')
+
+                if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
+                    if fixup_policy == 'warn':
+                        self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
+                            info_dict['id']))
+                    elif fixup_policy == 'detect_or_warn':
+                        fixup_pp = FFmpegFixupM4aPP(self)
+                        if fixup_pp.available:
+                            info_dict.setdefault('__postprocessors', [])
+                            info_dict['__postprocessors'].append(fixup_pp)
+                        else:
+                            self.report_warning(
+                                '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
+                                    info_dict['id']))
+                    else:
+                        assert fixup_policy in ('ignore', 'never')
 
                 try:
                     self.post_process(filename, info_dict)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2a1f8be0a..eb55d24ce 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -264,9 +264,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index f8507951c..0ffbca258 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -7,6 +7,7 @@ from .ffmpeg import (
     FFmpegEmbedSubtitlePP,
     FFmpegExtractAudioPP,
     FFmpegFixupStretchedPP,
+    FFmpegFixupM4aPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
     FFmpegVideoConvertorPP,
@@ -25,6 +26,7 @@ __all__ = [
     'FFmpegAudioFixPP',
     'FFmpegEmbedSubtitlePP',
     'FFmpegExtractAudioPP',
+    'FFmpegFixupM4aPP',
     'FFmpegFixupStretchedPP',
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index faccdc43d..855d1e6db 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -564,7 +564,7 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor):
     def run(self, info):
         stretched_ratio = info.get('stretched_ratio')
         if stretched_ratio is None or stretched_ratio == 1:
-            return
+            return True, info
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
@@ -577,3 +577,21 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         return True, info
+
+
+class FFmpegFixupM4aPP(FFmpegPostProcessor):
+    def run(self, info):
+        if info.get('container') != 'm4a_dash':
+            return True, info
+
+        filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
+
+        options = ['-c', 'copy', '-f', 'mp4']
+        self._downloader.to_screen('[ffmpeg] Correcting container in "%s"' % filename)
+        self.run_ffmpeg(filename, temp_filename, options)
+
+        os.remove(encodeFilename(filename))
+        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+
+        return True, info

From 6896a52721ce3a0dc4f33fa5dda508c30c56ce22 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 18:58:32 +0100
Subject: [PATCH 591/974] release 2015.01.23.4

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f39b94361..35f3e1b6b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.3'
+__version__ = '2015.01.23.4'

From 3e055aa5c3c069476b1ac9fc103dc1386cad763c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 23 Jan 2015 21:22:49 +0200
Subject: [PATCH 592/974] [cliphunter] Fix extraction and update test (Fixes
 #4362)

---
 youtube_dl/extractor/cliphunter.py | 56 ++++++++++++------------------
 1 file changed, 23 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py
index 2edab90a3..d46592cc5 100644
--- a/youtube_dl/extractor/cliphunter.py
+++ b/youtube_dl/extractor/cliphunter.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
+from ..utils import determine_ext
 
 
 _translation_table = {
@@ -27,10 +25,10 @@ class CliphunterIE(InfoExtractor):
     '''
     _TEST = {
         'url': 'http://www.cliphunter.com/w/1012420/Fun_Jynx_Maze_solo',
-        'md5': 'a2ba71eebf523859fe527a61018f723e',
+        'md5': 'b7c9bbd4eb3a226ab91093714dcaa480',
         'info_dict': {
             'id': '1012420',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Fun Jynx Maze solo',
             'thumbnail': 're:^https?://.*\.jpg$',
             'age_limit': 18,
@@ -44,39 +42,31 @@ class CliphunterIE(InfoExtractor):
         video_title = self._search_regex(
             r'mediaTitle = "([^"]+)"', webpage, 'title')
 
-        pl_fiji = self._search_regex(
-            r'pl_fiji = \'([^\']+)\'', webpage, 'video data')
-        pl_c_qual = self._search_regex(
-            r'pl_c_qual = "(.)"', webpage, 'video quality')
-        video_url = _decode(pl_fiji)
-        formats = [{
-            'url': video_url,
-            'format_id': 'default-%s' % pl_c_qual,
-        }]
+        fmts = {}
+        for fmt in ('mp4', 'flv'):
+            fmt_list = self._parse_json(self._search_regex(
+                r'var %sjson\s*=\s*(\[.*?\]);' % fmt, webpage, '%s formats' % fmt), video_id)
+            for f in fmt_list:
+                fmts[f['fname']] = _decode(f['sUrl'])
 
-        qualities_json = self._search_regex(
-            r'var pl_qualities\s*=\s*(.*?);\n', webpage, 'quality info')
-        qualities_data = json.loads(qualities_json)
+        qualities = self._parse_json(self._search_regex(
+            r'var player_btns\s*=\s*(.*?);\n', webpage, 'quality info'), video_id)
 
-        for i, t in enumerate(
-                re.findall(r"pl_fiji_([a-z0-9]+)\s*=\s*'([^']+')", webpage)):
-            quality_id, crypted_url = t
-            video_url = _decode(crypted_url)
+        formats = []
+        for fname, url in fmts.items():
             f = {
-                'format_id': quality_id,
-                'url': video_url,
-                'quality': i,
+                'url': url,
             }
-            if quality_id in qualities_data:
-                qd = qualities_data[quality_id]
-                m = re.match(
-                    r'''(?x)<b>(?P<width>[0-9]+)x(?P<height>[0-9]+)<\\/b>
-                        \s*\(\s*(?P<tbr>[0-9]+)\s*kb\\/s''', qd)
-                if m:
-                    f['width'] = int(m.group('width'))
-                    f['height'] = int(m.group('height'))
-                    f['tbr'] = int(m.group('tbr'))
+            if fname in qualities:
+                qual = qualities[fname]
+                f.update({
+                    'format_id': '%s_%sp' % (determine_ext(url), qual['h']),
+                    'width': qual['w'],
+                    'height': qual['h'],
+                    'tbr': qual['br'],
+                })
             formats.append(f)
+
         self._sort_formats(formats)
 
         thumbnail = self._search_regex(

From 48f00d15b1bdbe05be397849f6659fe5e0daa9c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 21:03:00 +0100
Subject: [PATCH 593/974] [auengine] Remove extractor

The test is probably infringing copyright and nobody has provided a new test (see #4643).
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/auengine.py | 50 --------------------------------
 2 files changed, 51 deletions(-)
 delete mode 100644 youtube_dl/extractor/auengine.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9e1ce5db3..3f7ca6f7d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -29,7 +29,6 @@ from .arte import (
 from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
-from .auengine import AUEngineIE
 from .azubu import AzubuIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
deleted file mode 100644
index a1b666be0..000000000
--- a/youtube_dl/extractor/auengine.py
+++ /dev/null
@@ -1,50 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse
-from ..utils import (
-    determine_ext,
-    ExtractorError,
-    remove_end,
-)
-
-
-class AUEngineIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?auengine\.com/embed\.php\?.*?file=(?P<id>[^&]+).*?'
-
-    _TEST = {
-        'url': 'http://auengine.com/embed.php?file=lfvlytY6&w=650&h=370',
-        'md5': '48972bdbcf1a3a2f5533e62425b41d4f',
-        'info_dict': {
-            'id': 'lfvlytY6',
-            'ext': 'mp4',
-            'title': '[Commie]The Legend of the Legendary Heroes - 03 - Replication Eye (Alpha Stigma)[F9410F5A]'
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(
-            r'<title>\s*(?P<title>.+?)\s*</title>', webpage, 'title')
-        video_urls = re.findall(r'http://\w+.auengine.com/vod/.*[^\W]', webpage)
-        video_url = compat_urllib_parse.unquote(video_urls[0])
-        thumbnails = re.findall(r'http://\w+.auengine.com/thumb/.*[^\W]', webpage)
-        thumbnail = compat_urllib_parse.unquote(thumbnails[0])
-
-        if not video_url:
-            raise ExtractorError('Could not find video URL')
-
-        ext = '.' + determine_ext(video_url)
-        title = remove_end(title, ext)
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': thumbnail,
-            'http_referer': 'http://www.auengine.com/flowplayer/flowplayer.commercial-3.2.14.swf',
-        }

From 12d1fb5aa9aabfe48676d6c60bbd114cc53a513f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 21:05:07 +0100
Subject: [PATCH 594/974] [twitch] PEP8

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 741df7cbc..87290d002 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -379,4 +379,4 @@ class TwitchStreamIE(TwitchBaseIE):
             'view_count': view_count,
             'formats': formats,
             'is_live': True,
-        }
\ No newline at end of file
+        }

From fdaaaaa878c42975936bf7f6ecf39a97436fefe2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 21:10:10 +0100
Subject: [PATCH 595/974] README: Recommend using flake8 instead of pyflake and
 pep8 separately

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 869523720..36b87444e 100644
--- a/README.md
+++ b/README.md
@@ -586,7 +586,7 @@ If you want to add support for a new site, you can follow this quick list (assum
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
-8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
+8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py

From a055469fafe088b6aa0e569d989cbf7f70535951 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 23:50:31 +0100
Subject: [PATCH 596/974] [downloader] Improve downloader selection

---
 youtube_dl/YoutubeDL.py           |  2 +-
 youtube_dl/downloader/__init__.py | 31 +++++++++++++++----------------
 youtube_dl/utils.py               | 22 ++++++++++++++++++++++
 3 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 521e4055e..e61e6c2a7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1179,7 +1179,7 @@ class YoutubeDL(object):
         if not self.params.get('skip_download', False):
             try:
                 def dl(name, info):
-                    fd = get_suitable_downloader(info)(self, self.params)
+                    fd = get_suitable_downloader(info, self.params)(self, self.params)
                     for ph in self._progress_hooks:
                         fd.add_progress_hook(ph)
                     if self.params.get('verbose'):
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 31e28df58..2aca3cab5 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -9,27 +9,26 @@ from .rtmp import RtmpFD
 from .f4m import F4mFD
 
 from ..utils import (
-    determine_ext,
+    determine_protocol,
 )
 
+PROTOCOL_MAP = {
+    'rtmp': RtmpFD,
+    'm3u8_native': NativeHlsFD,
+    'm3u8': HlsFD,
+    'mms': MplayerFD,
+    'rtsp': MplayerFD,
+    'f4m': F4mFD,
+}
 
-def get_suitable_downloader(info_dict):
+
+def get_suitable_downloader(info_dict, params={}):
     """Get the downloader class that can handle the info dict."""
-    url = info_dict['url']
-    protocol = info_dict.get('protocol')
+    protocol = determine_protocol(info_dict)
+    info_dict['protocol'] = protocol
+
+    return PROTOCOL_MAP.get(protocol, HttpFD)
 
-    if url.startswith('rtmp'):
-        return RtmpFD
-    if protocol == 'm3u8_native':
-        return NativeHlsFD
-    if (protocol == 'm3u8') or (protocol is None and determine_ext(url) == 'm3u8'):
-        return HlsFD
-    if url.startswith('mms') or url.startswith('rtsp'):
-        return MplayerFD
-    if determine_ext(url) == 'f4m':
-        return F4mFD
-    else:
-        return HttpFD
 
 __all__ = [
     'get_suitable_downloader',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 463cc20ff..2970d02a1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1642,3 +1642,25 @@ def is_html(first_bytes):
         s = first_bytes.decode('utf-8', 'replace')
 
     return re.match(r'^\s*<', s)
+
+
+def determine_protocol(info_dict):
+    protocol = info_dict.get('protocol')
+    if protocol is not None:
+        return protocol
+
+    url = info_dict['url']
+    if url.startswith('rtmp'):
+        return 'rtmp'
+    elif url.startswith('mms'):
+        return 'mms'
+    elif url.startswith('rtsp'):
+        return 'rtsp'
+
+    ext = determine_ext(url)
+    if ext == 'm3u8':
+        return 'm3u8'
+    elif ext == 'f4m':
+        return 'f4m'
+
+    return compat_urllib_parse_urlparse(url).scheme

From 222516d97d5ff9e62f3a9860fe2e65aa99c001b3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Jan 2015 01:38:48 +0100
Subject: [PATCH 597/974] [downloader] Lay groundwork for external downloaders.

This comes with a very simply implementation for wget; the real work is in setting up the infrastructure.
---
 youtube_dl/YoutubeDL.py           |   1 +
 youtube_dl/__init__.py            |   1 +
 youtube_dl/downloader/__init__.py |   9 +-
 youtube_dl/downloader/common.py   |  21 +++++
 youtube_dl/downloader/external.py | 131 ++++++++++++++++++++++++++++++
 youtube_dl/downloader/rtmp.py     |  14 +---
 youtube_dl/options.py             |   6 ++
 7 files changed, 169 insertions(+), 14 deletions(-)
 create mode 100644 youtube_dl/downloader/external.py

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e61e6c2a7..54e732943 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -219,6 +219,7 @@ class YoutubeDL(object):
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
+    external_downloader:  Executable of the external downloader to call.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 7bd7295e2..3fc7dc5c2 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -330,6 +330,7 @@ def _real_main(argv=None):
         'source_address': opts.source_address,
         'call_home': opts.call_home,
         'sleep_interval': opts.sleep_interval,
+        'external_downloader': opts.external_downloader,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 2aca3cab5..eff1122c5 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -1,12 +1,13 @@
 from __future__ import unicode_literals
 
 from .common import FileDownloader
+from .external import get_external_downloader
+from .f4m import F4mFD
 from .hls import HlsFD
 from .hls import NativeHlsFD
 from .http import HttpFD
 from .mplayer import MplayerFD
 from .rtmp import RtmpFD
-from .f4m import F4mFD
 
 from ..utils import (
     determine_protocol,
@@ -27,6 +28,12 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
+    external_downloader = params.get('external_downloader')
+    if external_downloader is not None:
+        ed = get_external_downloader(external_downloader)
+        if ed.supports(info_dict):
+            return ed
+
     return PROTOCOL_MAP.get(protocol, HttpFD)
 
 
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 82c917d92..c35c42c1d 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -325,3 +325,24 @@ class FileDownloader(object):
         # See YoutubeDl.py (search for progress_hooks) for a description of
         # this interface
         self._progress_hooks.append(ph)
+
+    def _debug_cmd(self, args, subprocess_encoding, exe=None):
+        if not self.params.get('verbose', False):
+            return
+
+        if exe is None:
+            exe = os.path.basename(args[0])
+
+        if subprocess_encoding:
+            str_args = [
+                a.decode(subprocess_encoding) if isinstance(a, bytes) else a
+                for a in args]
+        else:
+            str_args = args
+        try:
+            import pipes
+            shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
+        except ImportError:
+            shell_quote = repr
+        self.to_screen('[debug] %s command line: %s' % (
+            exe, shell_quote(str_args)))
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
new file mode 100644
index 000000000..c05596255
--- /dev/null
+++ b/youtube_dl/downloader/external.py
@@ -0,0 +1,131 @@
+from __future__ import unicode_literals
+
+import os.path
+import subprocess
+import sys
+
+from .common import FileDownloader
+from ..utils import (
+    encodeFilename,
+    std_headers,
+)
+
+
+class ExternalFD(FileDownloader):
+    def real_download(self, filename, info_dict):
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+
+        retval = self._call_downloader(tmpfilename, info_dict)
+        if retval == 0:
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))
+            self.try_rename(tmpfilename, filename)
+            self._hook_progress({
+                'downloaded_bytes': fsize,
+                'total_bytes': fsize,
+                'filename': filename,
+                'status': 'finished',
+            })
+            return True
+        else:
+            self.to_stderr('\n')
+            self.report_error('%s exited with code %d' % (
+                self.get_basename(), retval))
+            return False
+
+    @classmethod
+    def get_basename(cls):
+        return cls.__name__[:-2].lower()
+
+    @property
+    def exe(self):
+        return self.params.get('external_downloader')
+
+    @classmethod
+    def supports(cls, info_dict):
+        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
+
+    def _calc_headers(self, info_dict):
+        res = std_headers.copy()
+
+        ua = info_dict.get('user_agent')
+        if ua is not None:
+            res['User-Agent'] = ua
+
+        cookies = self._calc_cookies(info_dict)
+        if cookies:
+            res['Cookie'] = cookies
+
+        return res
+
+    def _calc_cookies(self, info_dict):
+        class _PseudoRequest(object):
+            def __init__(self, url):
+                self.url = url
+                self.headers = {}
+                self.unverifiable = False
+
+            def add_unredirected_header(self, k, v):
+                self.headers[k] = v
+
+            def get_full_url(self):
+                return self.url
+
+            def is_unverifiable(self):
+                return self.unverifiable
+
+            def has_header(self, h):
+                return h in self.headers
+
+        pr = _PseudoRequest(info_dict['url'])
+        self.ydl.cookiejar.add_cookie_header(pr)
+        return pr.headers.get('Cookie')
+
+    def _call_downloader(self, tmpfilename, info_dict):
+        """ Either overwrite this or implement _make_cmd """
+        cmd = self._make_cmd(tmpfilename, info_dict)
+
+        if sys.platform == 'win32' and sys.version_info < (3, 0):
+            # Windows subprocess module does not actually support Unicode
+            # on Python 2.x
+            # See http://stackoverflow.com/a/9951851/35070
+            subprocess_encoding = sys.getfilesystemencoding()
+            cmd = [a.encode(subprocess_encoding, 'ignore') for a in cmd]
+        else:
+            subprocess_encoding = None
+        self._debug_cmd(cmd, subprocess_encoding)
+
+        p = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        if p.returncode != 0:
+            self.to_stderr(stderr)
+        return p.returncode
+
+
+class WgetFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
+        for key, val in self._calc_headers(info_dict).items():
+            cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+
+_BY_NAME = dict(
+    (klass.get_basename(), klass)
+    for name, klass in globals().items()
+    if name.endswith('FD') and name != 'ExternalFD'
+)
+
+
+def list_external_downloaders():
+    return sorted(_BY_NAME.keys())
+
+
+def get_external_downloader(external_downloader):
+    """ Given the name of the executable, see whether we support the given
+        downloader . """
+    bn = os.path.basename(external_downloader)
+    return _BY_NAME[bn]
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 5346cb9a0..6dbbc053c 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -152,19 +152,7 @@ class RtmpFD(FileDownloader):
         else:
             subprocess_encoding = None
 
-        if self.params.get('verbose', False):
-            if subprocess_encoding:
-                str_args = [
-                    a.decode(subprocess_encoding) if isinstance(a, bytes) else a
-                    for a in args]
-            else:
-                str_args = args
-            try:
-                import pipes
-                shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
-            except ImportError:
-                shell_quote = repr
-            self.to_screen('[debug] rtmpdump command line: ' + shell_quote(str_args))
+        self._debug_cmd(args, subprocess_encoding, exe='rtmpdump')
 
         RD_SUCCESS = 0
         RD_FAILED = 1
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 262c60013..b38b8349f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -5,6 +5,7 @@ import optparse
 import shlex
 import sys
 
+from .downloader.external import list_external_downloaders
 from .compat import (
     compat_expanduser,
     compat_getenv,
@@ -389,6 +390,11 @@ def parseOpts(overrideArguments=None):
         '--playlist-reverse',
         action='store_true',
         help='Download playlist videos in reverse order')
+    downloader.add_option(
+        '--external-downloader',
+        dest='external_downloader', metavar='COMMAND',
+        help='(experimental) Use the specified external downloader. '
+             'Currently supports %s' % ','.join(list_external_downloaders()))
 
     workarounds = optparse.OptionGroup(parser, 'Workarounds')
     workarounds.add_option(

From 6aa4f54d6660bfb8fefe225e47cba7ade62fe564 Mon Sep 17 00:00:00 2001
From: Irfan Charania <irfan.charania@gmail.com>
Date: Fri, 23 Jan 2015 17:41:07 -0800
Subject: [PATCH 598/974] [videott] improve extraction

---
 youtube_dl/extractor/videott.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index 1f938838c..6f8084140 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -13,7 +13,7 @@ from ..utils import (
 class VideoTtIE(InfoExtractor):
     ID_NAME = 'video.tt'
     IE_DESC = 'video.tt - Your True Tube'
-    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:video/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
+    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:video\/|embed\/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
 
     _TEST = {
         'url': 'http://www.video.tt/watch_video.php?v=amd5YujV8',

From fc2d6abfe7e7a216825296001b889e5455e0412f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Jan 2015 16:11:40 +0600
Subject: [PATCH 599/974] [videott] Improve _VALID_URL and add test

---
 youtube_dl/extractor/videott.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index 6f8084140..ececc7ee0 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -13,9 +13,9 @@ from ..utils import (
 class VideoTtIE(InfoExtractor):
     ID_NAME = 'video.tt'
     IE_DESC = 'video.tt - Your True Tube'
-    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:video\/|embed\/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
+    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:(?:video|embed)/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.video.tt/watch_video.php?v=amd5YujV8',
         'md5': 'b13aa9e2f267effb5d1094443dff65ba',
         'info_dict': {
@@ -26,7 +26,10 @@ class VideoTtIE(InfoExtractor):
             'upload_date': '20130827',
             'uploader': 'joseph313',
         }
-    }
+    }, {
+        'url': 'http://video.tt/embed/amd5YujV8',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From b95aab8482881e8b1f7fba856da816a2dbc50d0f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Jan 2015 11:42:20 +0100
Subject: [PATCH 600/974] [youtube:truncated_url] Add x-yt-cl URLs (#4773)

---
 youtube_dl/extractor/youtube.py | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eb55d24ce..b7b91f354 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1682,11 +1682,17 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     IE_NAME = 'youtube:truncated_url'
     IE_DESC = False  # Do not list
     _VALID_URL = r'''(?x)
-        (?:https?://)?[^/]+/watch\?(?:
+        (?:https?://)?
+        (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
+        (?:watch\?(?:
             feature=[a-z_]+|
-            annotation_id=annotation_[^&]+
-        )?$|
-        (?:https?://)?(?:www\.)?youtube\.com/attribution_link\?a=[^&]+$
+            annotation_id=annotation_[^&]+|
+            x-yt-cl=[0-9]+|
+        )?
+        |
+            attribution_link\?a=[^&]+
+        )
+        $
     '''
 
     _TESTS = [{
@@ -1695,6 +1701,12 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     }, {
         'url': 'http://www.youtube.com/watch?',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?feature=foo',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -1710,7 +1722,7 @@ class YoutubeTruncatedURLIE(InfoExtractor):
 class YoutubeTruncatedIDIE(InfoExtractor):
     IE_NAME = 'youtube:truncated_id'
     IE_DESC = False  # Do not list
-    _VALID_URL = r'https?://(?:www\.)youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
 
     _TESTS = [{
         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',

From 384b62028a4c3c35727d714ccfc9944a36934069 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Jan 2015 13:33:45 +0100
Subject: [PATCH 601/974] [downloader/external] Add curl and aria2c (Closes
 #182)

---
 youtube_dl/downloader/external.py | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index c05596255..7ebe40096 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -97,13 +97,22 @@ class ExternalFD(FileDownloader):
         self._debug_cmd(cmd, subprocess_encoding)
 
         p = subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = p.communicate()
+            cmd, stderr=subprocess.PIPE)
+        _, stderr = p.communicate()
         if p.returncode != 0:
             self.to_stderr(stderr)
         return p.returncode
 
 
+class CurlFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-o', tmpfilename]
+        for key, val in self._calc_headers(info_dict).items():
+            cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+
 class WgetFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
@@ -113,6 +122,20 @@ class WgetFD(ExternalFD):
         return cmd
 
 
+class Aria2cFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [
+            self.exe, '-c',
+            '--min-split-size', '1M', '--max-connection-per-server', '4']
+        dn = os.path.dirname(tmpfilename)
+        if dn:
+            cmd += ['--dir', dn]
+        cmd += ['--out', os.path.basename(tmpfilename)]
+        for key, val in self._calc_headers(info_dict).items():
+            cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += ['--', info_dict['url']]
+        return cmd
+
 _BY_NAME = dict(
     (klass.get_basename(), klass)
     for name, klass in globals().items()

From 3fcfb8e9faf3cf1dcadedd6fecc5158a86d07065 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:07:21 +0100
Subject: [PATCH 602/974] [utils] YoutubeDLHandler: don't use
 'Youtubedl-user-agent' for overriding the default user agent

Setting the 'User-Agent' header is enough
---
 youtube_dl/downloader/http.py       | 2 +-
 youtube_dl/extractor/atresplayer.py | 2 +-
 youtube_dl/extractor/bliptv.py      | 2 +-
 youtube_dl/extractor/mtv.py         | 2 +-
 youtube_dl/utils.py                 | 5 -----
 5 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index e68f20c9f..90a2e4c53 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -25,7 +25,7 @@ class HttpFD(FileDownloader):
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
         if 'user_agent' in info_dict:
-            headers['Youtubedl-user-agent'] = info_dict['user_agent']
+            headers['User-agent'] = info_dict['user_agent']
         if 'http_referer' in info_dict:
             headers['Referer'] = info_dict['http_referer']
         add_headers = info_dict.get('http_headers')
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index f42862be3..f016368fa 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -95,7 +95,7 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
         for fmt in ['windows', 'android_tablet']:
             request = compat_urllib_request.Request(
                 self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
-            request.add_header('Youtubedl-user-agent', self._USER_AGENT)
+            request.add_header('User-Agent', self._USER_AGENT)
 
             fmt_json = self._download_json(
                 request, video_id, 'Downloading %s video JSON' % fmt)
diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index 14b814120..436cc5155 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -199,7 +199,7 @@ class BlipTVIE(SubtitlesInfoExtractor):
         # For some weird reason, blip.tv serves a video instead of subtitles
         # when we request with a common UA
         req = compat_urllib_request.Request(url)
-        req.add_header('Youtubedl-user-agent', 'youtube-dl')
+        req.add_header('User-Agent', 'youtube-dl')
         return self._download_webpage(req, None, note=False)
 
 
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 5ebc78033..22a726327 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -53,7 +53,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         webpage_url = self._MOBILE_TEMPLATE % mtvn_id
         req = compat_urllib_request.Request(webpage_url)
         # Otherwise we get a webpage that would execute some javascript
-        req.add_header('Youtubedl-user-agent', 'curl/7')
+        req.add_header('User-Agent', 'curl/7')
         webpage = self._download_webpage(req, mtvn_id,
                                          'Downloading mobile page')
         metrics_url = unescapeHTML(self._search_regex(r'<a href="(http://metrics.+?)"', webpage, 'url'))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2970d02a1..d22b03134 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -606,11 +606,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             if 'Accept-encoding' in req.headers:
                 del req.headers['Accept-encoding']
             del req.headers['Youtubedl-no-compression']
-        if 'Youtubedl-user-agent' in req.headers:
-            if 'User-agent' in req.headers:
-                del req.headers['User-agent']
-            req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
-            del req.headers['Youtubedl-user-agent']
 
         if sys.version_info < (2, 7) and '#' in req.get_full_url():
             # Python 2.6 is brain-dead when it comes to fragments

From e1554a407d8953d0ec4fabec7f5a3b3906d1029b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:19:58 +0100
Subject: [PATCH 603/974] [extractors] Use http_headers for setting the
 User-Agent and the Referer

---
 youtube_dl/downloader/http.py         | 4 ----
 youtube_dl/extractor/appletrailers.py | 4 +++-
 youtube_dl/extractor/common.py        | 1 -
 youtube_dl/extractor/videomega.py     | 4 +++-
 youtube_dl/extractor/wdr.py           | 4 +++-
 5 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 90a2e4c53..4db50ee90 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -24,10 +24,6 @@ class HttpFD(FileDownloader):
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
-        if 'user_agent' in info_dict:
-            headers['User-agent'] = info_dict['user_agent']
-        if 'http_referer' in info_dict:
-            headers['Referer'] = info_dict['http_referer']
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 7cd0482c7..70621946d 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -129,7 +129,9 @@ class AppleTrailersIE(InfoExtractor):
                 'thumbnail': thumbnail,
                 'upload_date': upload_date,
                 'uploader_id': uploader_id,
-                'user_agent': 'QuickTime compatible (youtube-dl)',
+                'http_headers': {
+                    'User-Agent': 'QuickTime compatible (youtube-dl)',
+                },
             })
 
         return {
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 03f3f18c8..523400062 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -108,7 +108,6 @@ class InfoExtractor(object):
                                   (quality takes higher priority)
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
-                    * http_referer  HTTP Referer header value to set.
                     * http_method  HTTP method to use for the download.
                     * http_headers  A dictionary of additional HTTP headers
                                  to add to the request.
diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index fc6e05fe0..273030316 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -62,5 +62,7 @@ class VideoMegaIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
-            'http_referer': iframe_url,
+            'http_headers': {
+                'Referer': iframe_url,
+            },
         }
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 45466e31b..313b9c15d 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -169,7 +169,9 @@ class WDRMobileIE(InfoExtractor):
             'title': mobj.group('title'),
             'age_limit': int(mobj.group('age_limit')),
             'url': url,
-            'user_agent': 'mobile',
+            'http_headers': {
+                'User-Agent': 'mobile',
+            },
         }
 
 

From 587a9c27496979d983296944fee8d1fa589e1b85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:25:09 +0100
Subject: [PATCH 604/974] [downloader/external] Use the 'http_headers' field

---
 youtube_dl/downloader/external.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 7ebe40096..5bf24ccbb 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -49,9 +49,9 @@ class ExternalFD(FileDownloader):
     def _calc_headers(self, info_dict):
         res = std_headers.copy()
 
-        ua = info_dict.get('user_agent')
-        if ua is not None:
-            res['User-Agent'] = ua
+        add_headers = info_dict.get('http_headers')
+        if add_headers:
+            res.update(add_headers)
 
         cookies = self._calc_cookies(info_dict)
         if cookies:

From 8011fba3ae4301995f77c64ac63a4a467ef73b7e Mon Sep 17 00:00:00 2001
From: David-Development <david-dev@live.de>
Date: Sat, 24 Jan 2015 18:28:16 +0100
Subject: [PATCH 605/974] [rtl2] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rtl2.py     | 98 ++++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)
 create mode 100644 youtube_dl/extractor/rtl2.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3f7ca6f7d..03c56156a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -349,6 +349,7 @@ from .rtbf import RTBFIE
 from .rte import RteIE
 from .rtlnl import RtlXlIE
 from .rtlnow import RTLnowIE
+from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
new file mode 100644
index 000000000..2bdbf2ec2
--- /dev/null
+++ b/youtube_dl/extractor/rtl2.py
@@ -0,0 +1,98 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    unified_strdate,
+    int_or_none,
+)
+
+
+class RTL2IE(InfoExtractor):
+    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
+    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))'
+    _TEST = {
+        'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
+        'md5': 'dsadasdada',
+        'info_dict': {
+            'id': 'folge-203-0',
+            'ext': 'f4v',
+            'title': 'GRIP sucht den Sommerk\xf6nig',
+	    'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+            # TODO more properties, either as:
+            # * A value
+            # * MD5 checksum; start the string with md5:
+            # * A regular expression; start the string with re:
+            # * Any Python type (for example int or float)
+        },
+	#'params': {
+                # rtmp download
+        #	'skip_download': True,
+	#},
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_page_url = 'http://%s/' % mobj.group('domain')
+        video_id = mobj.group('video_id')
+	
+        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
+
+	vico_id = self._html_search_regex(r'vico_id: ([0-9]+)', webpage, '%s');
+	vivi_id = self._html_search_regex(r'vivi_id: ([0-9]+)', webpage, '%s');
+
+	info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
+	webpage = self._download_webpage(info_url, '')
+
+	video_info = json.loads(webpage.decode("latin1"))
+	print video_info
+
+
+	#self._download_webpage('http://cp108781.edgefcs.net/crossdomain.xml', '')
+
+	download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s');
+	title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s');
+	description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s');
+	#ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s');
+
+	thumbnail = video_info["video"]["image"]
+
+	download_url = download_url.replace("\\", "")
+
+	stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s');
+
+	#upload_date = self._html_search_regex(r'property=\"dc:date\".*?datatype=\"xsd:dateTime\".*?content=\"(.*?)\"', webpage, 'title')
+	#download_url += " -y " + stream_url
+	
+	#print stream_url
+	#print download_url
+	#print description
+	#print title
+	#print ext
+
+	formats = []
+
+	fmt = {
+	    'url' : download_url,
+            #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
+            'play_path': stream_url,
+            #'player_url': 'http://www.cbsnews.com/[[IMPORT]]/vidtech.cbsinteractive.com/player/3_3_0/CBSI_PLAYER_HD.swf',
+            #'page_url': 'http://www.cbsnews.com',
+            #'ext': ext,	
+        }
+
+	formats.append(fmt)
+
+
+        return {
+	    'id': video_id,
+            'title': title,
+	    'thumbnail' : thumbnail,
+	    'description' : description,
+            'formats': formats,
+        }

From e5660ee6aebacee4b149648bee1a7ce1ec72c1e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:52:26 +0100
Subject: [PATCH 606/974] [YoutubeDL] Fill the info dict 'http_headers' field
 with all the headers available

Useful for external tools using the json output.

The methods '_calc_headers' and '_calc_cookies' have been copied from the downloader/external, now they just use "info_dict['http_headers']".
---
 youtube_dl/YoutubeDL.py           | 36 ++++++++++++++++++++++++++
 youtube_dl/downloader/external.py | 43 +++----------------------------
 2 files changed, 39 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 54e732943..4c3d45f48 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -56,6 +56,7 @@ from .utils import (
     preferredencoding,
     SameFileError,
     sanitize_filename,
+    std_headers,
     subtitles_filename,
     takewhile_inclusive,
     UnavailableVideoError,
@@ -865,6 +866,36 @@ class YoutubeDL(object):
                 return matches[-1]
         return None
 
+    def _calc_headers(self, info_dict):
+        res = std_headers.copy()
+
+        add_headers = info_dict.get('http_headers')
+        if add_headers:
+            res.update(add_headers)
+
+        cookies = self._calc_cookies(info_dict)
+        if cookies:
+            res['Cookie'] = cookies
+
+        return res
+
+    def _calc_cookies(self, info_dict):
+        class _PseudoRequest(object):
+            def __init__(self, url):
+                self.url = url
+                self.headers = {}
+                self.unverifiable = False
+
+            def add_unredirected_header(self, k, v):
+                self.headers[k] = v
+
+            def get_full_url(self):
+                return self.url
+
+        pr = _PseudoRequest(info_dict['url'])
+        self.cookiejar.add_cookie_header(pr)
+        return pr.headers.get('Cookie')
+
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'
 
@@ -933,6 +964,11 @@ class YoutubeDL(object):
             # Automatically determine file extension if missing
             if 'ext' not in format:
                 format['ext'] = determine_ext(format['url']).lower()
+            # Add HTTP headers, so that external programs can use them from the
+            # json output
+            full_format_info = info_dict.copy()
+            full_format_info.update(format)
+            format['http_headers'] = self._calc_headers(full_format_info)
 
         format_limit = self.params.get('format_limit', None)
         if format_limit:
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 5bf24ccbb..af9fdba75 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -7,7 +7,6 @@ import sys
 from .common import FileDownloader
 from ..utils import (
     encodeFilename,
-    std_headers,
 )
 
 
@@ -46,42 +45,6 @@ class ExternalFD(FileDownloader):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
-    def _calc_headers(self, info_dict):
-        res = std_headers.copy()
-
-        add_headers = info_dict.get('http_headers')
-        if add_headers:
-            res.update(add_headers)
-
-        cookies = self._calc_cookies(info_dict)
-        if cookies:
-            res['Cookie'] = cookies
-
-        return res
-
-    def _calc_cookies(self, info_dict):
-        class _PseudoRequest(object):
-            def __init__(self, url):
-                self.url = url
-                self.headers = {}
-                self.unverifiable = False
-
-            def add_unredirected_header(self, k, v):
-                self.headers[k] = v
-
-            def get_full_url(self):
-                return self.url
-
-            def is_unverifiable(self):
-                return self.unverifiable
-
-            def has_header(self, h):
-                return h in self.headers
-
-        pr = _PseudoRequest(info_dict['url'])
-        self.ydl.cookiejar.add_cookie_header(pr)
-        return pr.headers.get('Cookie')
-
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
         cmd = self._make_cmd(tmpfilename, info_dict)
@@ -107,7 +70,7 @@ class ExternalFD(FileDownloader):
 class CurlFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-o', tmpfilename]
-        for key, val in self._calc_headers(info_dict).items():
+        for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += ['--', info_dict['url']]
         return cmd
@@ -116,7 +79,7 @@ class CurlFD(ExternalFD):
 class WgetFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
-        for key, val in self._calc_headers(info_dict).items():
+        for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += ['--', info_dict['url']]
         return cmd
@@ -131,7 +94,7 @@ class Aria2cFD(ExternalFD):
         if dn:
             cmd += ['--dir', dn]
         cmd += ['--out', os.path.basename(tmpfilename)]
-        for key, val in self._calc_headers(info_dict).items():
+        for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += ['--', info_dict['url']]
         return cmd

From 4b405cfc6e4f9184567b94ee2f8a4a9851a912b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 20:05:35 +0100
Subject: [PATCH 607/974] [YoutubeDL._calc_cookies] Restore the 'has_header'
 method

I didn't copied it from downloader/external
---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4c3d45f48..458fd15ea 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -892,6 +892,9 @@ class YoutubeDL(object):
             def get_full_url(self):
                 return self.url
 
+            def has_header(self, h):
+                return h in self.headers
+
         pr = _PseudoRequest(info_dict['url'])
         self.cookiejar.add_cookie_header(pr)
         return pr.headers.get('Cookie')

From 1070711d6003e6750b8cf803c3926b2e273a9e85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 20:12:47 +0100
Subject: [PATCH 608/974] [YoutubeDL._calc_cookies] Restore the
 'is_unverifiable'

I should have check everything was copied before commiting 4b405cfc6e4f9184567b94ee2f8a4a9851a912b5.
---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 458fd15ea..d6728b2dd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -892,6 +892,9 @@ class YoutubeDL(object):
             def get_full_url(self):
                 return self.url
 
+            def is_unverifiable(self):
+                return self.unverifiable
+
             def has_header(self, h):
                 return h in self.headers
 

From 7906d199a11c0f8deccc0f87d7ee850106c361dc Mon Sep 17 00:00:00 2001
From: David-Development <david-dev@live.de>
Date: Sat, 24 Jan 2015 18:28:16 +0100
Subject: [PATCH 609/974] [rtl2] Add new extractor

---
 youtube_dl/downloader/rtmp.py    |   8 ++-
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/rtl2.py     | 100 +++++++++++++++++++++++++++++++
 3 files changed, 107 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/rtl2.py

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6dbbc053c..6f7ad588e 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -104,7 +104,8 @@ class RtmpFD(FileDownloader):
         live = info_dict.get('rtmp_live', False)
         conn = info_dict.get('rtmp_conn', None)
         protocol = info_dict.get('rtmp_protocol', None)
-
+        no_resume = info_dict.get('no_resume', False)
+        
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
         test = self.params.get('test', False)
@@ -141,7 +142,10 @@ class RtmpFD(FileDownloader):
             basic_args += ['--conn', conn]
         if protocol is not None:
             basic_args += ['--protocol', protocol]
-        args = basic_args + [[], ['--resume', '--skip', '1']][not live and self.params.get('continuedl', False)]
+        if not no_resume:
+            basic_args += ['--resume']
+        
+        args = basic_args + [[], ['--skip', '1']][not live and self.params.get('continuedl', False)]
 
         if sys.platform == 'win32' and sys.version_info < (3, 0):
             # Windows subprocess module does not actually support Unicode
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3f7ca6f7d..03c56156a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -349,6 +349,7 @@ from .rtbf import RTBFIE
 from .rte import RteIE
 from .rtlnl import RtlXlIE
 from .rtlnow import RTLnowIE
+from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
new file mode 100644
index 000000000..14b45e86e
--- /dev/null
+++ b/youtube_dl/extractor/rtl2.py
@@ -0,0 +1,100 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    unified_strdate,
+    int_or_none,
+)
+
+
+class RTL2IE(InfoExtractor):
+    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
+    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))'
+    _TESTS = [{
+            'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
+            'info_dict': {
+                'id': 'folge-203-0',
+                'ext': 'f4v',
+                'title': 'GRIP sucht den Sommerk\xf6nig',
+                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+            },
+            'params': {
+                # rtmp download
+                #'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
+            'info_dict': {
+                'id': '21040-anna-erwischt-alex',
+                'ext': 'f4v',
+                'title': 'GRIP sucht den Sommerk\xf6nig',
+                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+            },
+            'params': {
+                # rtmp download
+                #'skip_download': True,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_page_url = 'http://%s/' % mobj.group('domain')
+        video_id = mobj.group('video_id')
+    
+        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
+
+        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, '%s');
+        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, '%s');
+
+        info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
+        webpage = self._download_webpage(info_url, '')
+
+        video_info = json.loads(webpage)
+
+        download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s');
+        title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s');
+        description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s');
+        #ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s');
+
+        thumbnail = video_info["video"]["image"]
+
+        download_url = download_url.replace("\\", "")
+
+        stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s')
+
+        #print(download_url)
+        #print(stream_url)
+        #print(title)
+        #print(description)
+        #print(video_id)
+        
+        formats = []
+
+        fmt = {
+            'url' : download_url,
+                #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
+                'play_path': stream_url,
+                'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
+                'page_url': url,
+                'flash_version' : "LNX 11,2,202,429",
+                'rtmp_conn' : ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"],
+                'no_resume' : 1,
+        }
+
+        formats.append(fmt)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail' : thumbnail,
+            'description' : description,
+            'formats': formats,
+        }

From 1e108029907ca28b75f37d2cf0bf25bcabbfbdac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jan 2015 05:21:39 +0600
Subject: [PATCH 610/974] [krasview] Fix extraction

---
 youtube_dl/extractor/krasview.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/krasview.py b/youtube_dl/extractor/krasview.py
index 6f3d2345b..e46954b47 100644
--- a/youtube_dl/extractor/krasview.py
+++ b/youtube_dl/extractor/krasview.py
@@ -2,18 +2,17 @@
 from __future__ import unicode_literals
 
 import json
-import re
 
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    unescapeHTML,
+    js_to_json,
 )
 
 
 class KrasViewIE(InfoExtractor):
     IE_DESC = 'Красвью'
-    _VALID_URL = r'https?://krasview\.ru/video/(?P<id>\d+)'
+    _VALID_URL = r'https?://krasview\.ru/(?:video|embed)/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://krasview.ru/video/512228',
@@ -29,20 +28,18 @@ class KrasViewIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        flashvars = json.loads(self._search_regex(
-            r'flashvars\s*:\s*({.+?})\s*}\);', webpage, 'flashvars'))
+        flashvars = json.loads(js_to_json(self._search_regex(
+            r'video_Init\(({.+?})', webpage, 'flashvars')))
 
         video_url = flashvars['url']
-        title = unescapeHTML(flashvars['title'])
-        description = unescapeHTML(flashvars.get('subtitle') or self._og_search_description(webpage, default=None))
-        thumbnail = flashvars['image']
-        duration = int(flashvars['duration'])
-        filesize = int(flashvars['size'])
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage, default=None)
+        thumbnail = flashvars.get('image') or self._og_search_thumbnail(webpage)
+        duration = int_or_none(flashvars.get('duration'))
         width = int_or_none(self._og_search_property('video:width', webpage, 'video width'))
         height = int_or_none(self._og_search_property('video:height', webpage, 'video height'))
 
@@ -53,7 +50,6 @@ class KrasViewIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'duration': duration,
-            'filesize': filesize,
             'width': width,
             'height': height,
         }

From cfb56d1af38f3e1e0251dbd8a20e3ed8884976ff Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 02:38:47 +0100
Subject: [PATCH 611/974] Add --list-thumbnails

---
 test/test_utils.py               | 11 ++++++++++
 youtube_dl/YoutubeDL.py          | 37 ++++++++++++++++++++++++++++----
 youtube_dl/__init__.py           |  1 +
 youtube_dl/extractor/common.py   |  2 ++
 youtube_dl/extractor/testtube.py | 16 ++++++++++++--
 youtube_dl/options.py            | 15 +++++++++----
 youtube_dl/utils.py              |  8 +++++++
 7 files changed, 80 insertions(+), 10 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index bdd7f268a..ebec7986f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -52,6 +52,7 @@ from youtube_dl.utils import (
     urlencode_postdata,
     version_tuple,
     xpath_with_ns,
+    render_table,
 )
 
 
@@ -434,5 +435,15 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
         self.assertTrue(is_html(  # UTF-32-LE
             b'\xFF\xFE\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4\x00\x00\x00'))
 
+    def test_render_table(self):
+        self.assertEqual(
+            render_table(
+                ['a', 'bcd'],
+                [[123, 4], [9999, 51]]),
+            'a    bcd\n'
+            '123  4\n'
+            '9999 51')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d6728b2dd..e0f5a0d74 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -54,6 +54,7 @@ from .utils import (
     PostProcessingError,
     platform_name,
     preferredencoding,
+    render_table,
     SameFileError,
     sanitize_filename,
     std_headers,
@@ -221,6 +222,8 @@ class YoutubeDL(object):
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
     external_downloader:  Executable of the external downloader to call.
+    listformats:       Print an overview of available video formats and exit.
+    list_thumbnails:   Print a table of all thumbnails and exit.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -916,9 +919,14 @@ class YoutubeDL(object):
             info_dict['playlist_index'] = None
 
         thumbnails = info_dict.get('thumbnails')
+        if thumbnails is None:
+            thumbnail = info_dict.get('thumbnail')
+            if thumbnail:
+                thumbnails = [{'url': thumbnail}]
         if thumbnails:
             thumbnails.sort(key=lambda t: (
-                t.get('width'), t.get('height'), t.get('url')))
+                t.get('preference'), t.get('width'), t.get('height'),
+                t.get('id'), t.get('url')))
             for t in thumbnails:
                 if 'width' in t and 'height' in t:
                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
@@ -990,9 +998,12 @@ class YoutubeDL(object):
             # element in the 'formats' field in info_dict is info_dict itself,
             # wich can't be exported to json
             info_dict['formats'] = formats
-        if self.params.get('listformats', None):
+        if self.params.get('listformats'):
             self.list_formats(info_dict)
             return
+        if self.params.get('list_thumbnails'):
+            self.list_thumbnails(info_dict)
+            return
 
         req_format = self.params.get('format')
         if req_format is None:
@@ -1500,8 +1511,26 @@ class YoutubeDL(object):
         header_line = line({
             'format_id': 'format code', 'ext': 'extension',
             'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
-        self.to_screen('[info] Available formats for %s:\n%s\n%s' %
-                       (info_dict['id'], header_line, '\n'.join(formats_s)))
+        self.to_screen(
+            '[info] Available formats for %s:\n%s\n%s' %
+            (info_dict['id'], header_line, '\n'.join(formats_s)))
+
+    def list_thumbnails(self, info_dict):
+        thumbnails = info_dict.get('thumbnails')
+        if not thumbnails:
+            tn_url = info_dict.get('thumbnail')
+            if tn_url:
+                thumbnails = [{'id': '0', 'url': tn_url}]
+            else:
+                self.to_screen(
+                    '[info] No thumbnails present for %s' % info_dict['id'])
+                return
+
+        self.to_screen(
+            '[info] Thumbnails for %s:' % info_dict['id'])
+        self.to_screen(render_table(
+            ['ID', 'width', 'height', 'URL'],
+            [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
 
     def urlopen(self, req):
         """ Start an HTTP download """
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 3fc7dc5c2..a3f82612c 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -331,6 +331,7 @@ def _real_main(argv=None):
         'call_home': opts.call_home,
         'sleep_interval': opts.sleep_interval,
         'external_downloader': opts.external_downloader,
+        'list_thumbnails': opts.list_thumbnails,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 523400062..7b7a832dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -129,7 +129,9 @@ class InfoExtractor(object):
                     something like "4234987", title "Dancing naked mole rats",
                     and display_id "dancing-naked-mole-rats"
     thumbnails:     A list of dictionaries, with the following entries:
+                        * "id" (optional, string) - Thumbnail format ID
                         * "url"
+                        * "preference" (optional, int) - quality of the image
                         * "width" (optional, int)
                         * "height" (optional, int)
                         * "resolution" (optional, string "{width}x{height"},
diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
index fd47e71a2..6a7b5e49d 100644
--- a/youtube_dl/extractor/testtube.py
+++ b/youtube_dl/extractor/testtube.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    qualities,
+)
 
 
 class TestTubeIE(InfoExtractor):
@@ -46,13 +49,22 @@ class TestTubeIE(InfoExtractor):
         self._sort_formats(formats)
 
         duration = int_or_none(info.get('duration'))
+        images = info.get('images')
+        thumbnails = None
+        preference = qualities(['mini', 'small', 'medium', 'large'])
+        if images:
+            thumbnails = [{
+                'id': thumbnail_id,
+                'url': img_url,
+                'preference': preference(thumbnail_id)
+            } for thumbnail_id, img_url in images.items()]
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': info['title'],
             'description': info.get('summary'),
-            'thumbnail': info.get('images', {}).get('large'),
+            'thumbnails': thumbnails,
             'uploader': info.get('show', {}).get('name'),
             'uploader_id': info.get('show', {}).get('slug'),
             'duration': duration,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index b38b8349f..e3b4b8a8a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -614,10 +614,6 @@ def parseOpts(overrideArguments=None):
         '--write-annotations',
         action='store_true', dest='writeannotations', default=False,
         help='write video annotations to a .annotation file')
-    filesystem.add_option(
-        '--write-thumbnail',
-        action='store_true', dest='writethumbnail', default=False,
-        help='write thumbnail image to disk')
     filesystem.add_option(
         '--load-info',
         dest='load_info_filename', metavar='FILE',
@@ -637,6 +633,16 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='rm_cachedir',
         help='Delete all filesystem cache files')
 
+    thumbnail = optparse.OptionGroup(parser, 'Thumbnail images')
+    thumbnail.add_option(
+        '--write-thumbnail',
+        action='store_true', dest='writethumbnail', default=False,
+        help='write thumbnail image to disk')
+    thumbnail.add_option(
+        '--list-thumbnails',
+        action='store_true', dest='list_thumbnails', default=False,
+        help='Simulate and list all available thumbnail formats')
+
     postproc = optparse.OptionGroup(parser, 'Post-processing Options')
     postproc.add_option(
         '-x', '--extract-audio',
@@ -702,6 +708,7 @@ def parseOpts(overrideArguments=None):
     parser.add_option_group(selection)
     parser.add_option_group(downloader)
     parser.add_option_group(filesystem)
+    parser.add_option_group(thumbnail)
     parser.add_option_group(verbosity)
     parser.add_option_group(workarounds)
     parser.add_option_group(video_format)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d22b03134..b8c52af74 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1659,3 +1659,11 @@ def determine_protocol(info_dict):
         return 'f4m'
 
     return compat_urllib_parse_urlparse(url).scheme
+
+
+def render_table(header_row, data):
+    """ Render a list of rows, each as a list of values """
+    table = [header_row] + data
+    max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
+    format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
+    return '\n'.join(format_str % tuple(row) for row in table)

From ec82d85acdc497436e7e2e767088d3ecb5947f68 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 03:11:12 +0100
Subject: [PATCH 612/974] [YoutubeDL] Implement --write-all-thumbnails (Closes
 #2269)

---
 youtube_dl/YoutubeDL.py | 57 +++++++++++++++++++++++++++--------------
 youtube_dl/__init__.py  |  1 +
 youtube_dl/options.py   |  4 +++
 3 files changed, 43 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e0f5a0d74..0e73dc8ff 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -146,6 +146,7 @@ class YoutubeDL(object):
     writeinfojson:     Write the video description to a .info.json file
     writeannotations:  Write the video annotations to a .annotations.xml file
     writethumbnail:    Write the thumbnail image to a file
+    write_all_thumbnails:  Write all thumbnail formats to files
     writesubtitles:    Write the video subtitles to a file
     writeautomaticsub: Write the automatic subtitles to a file
     allsubtitles:      Downloads all the subtitles of the video
@@ -1210,25 +1211,7 @@ class YoutubeDL(object):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return
 
-        if self.params.get('writethumbnail', False):
-            if info_dict.get('thumbnail') is not None:
-                thumb_format = determine_ext(info_dict['thumbnail'], 'jpg')
-                thumb_filename = os.path.splitext(filename)[0] + '.' + thumb_format
-                if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
-                    self.to_screen('[%s] %s: Thumbnail is already present' %
-                                   (info_dict['extractor'], info_dict['id']))
-                else:
-                    self.to_screen('[%s] %s: Downloading thumbnail ...' %
-                                   (info_dict['extractor'], info_dict['id']))
-                    try:
-                        uf = self.urlopen(info_dict['thumbnail'])
-                        with open(thumb_filename, 'wb') as thumbf:
-                            shutil.copyfileobj(uf, thumbf)
-                        self.to_screen('[%s] %s: Writing thumbnail to: %s' %
-                                       (info_dict['extractor'], info_dict['id'], thumb_filename))
-                    except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                        self.report_warning('Unable to download thumbnail "%s": %s' %
-                                            (info_dict['thumbnail'], compat_str(err)))
+        self._write_thumbnails(info_dict, filename)
 
         if not self.params.get('skip_download', False):
             try:
@@ -1676,3 +1659,39 @@ class YoutubeDL(object):
         if encoding is None:
             encoding = preferredencoding()
         return encoding
+
+    def _write_thumbnails(self, info_dict, filename):
+        if self.params.get('writethumbnail', False):
+            thumbnails = info_dict.get('thumbnails')
+            if thumbnails:
+                thumbnails = [thumbnails[-1]]
+        elif self.params.get('write_all_thumbnails', False):
+            thumbnails = info_dict.get('thumbnails')
+        else:
+            return
+
+        if not thumbnails:
+            # No thumbnails present, so return immediately
+            return
+
+        for t in thumbnails:
+            thumb_ext = determine_ext(t['url'], 'jpg')
+            suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
+            thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
+            thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
+
+            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
+                self.to_screen('[%s] %s: Thumbnail %sis already present' %
+                               (info_dict['extractor'], info_dict['id'], thumb_display_id))
+            else:
+                self.to_screen('[%s] %s: Downloading thumbnail %s...' %
+                               (info_dict['extractor'], info_dict['id'], thumb_display_id))
+                try:
+                    uf = self.urlopen(t['url'])
+                    with open(thumb_filename, 'wb') as thumbf:
+                        shutil.copyfileobj(uf, thumbf)
+                    self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
+                                   (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
+                except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                    self.report_warning('Unable to download thumbnail "%s": %s' %
+                                        (t['url'], compat_str(err)))
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index a3f82612c..04f668334 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -286,6 +286,7 @@ def _real_main(argv=None):
         'writeannotations': opts.writeannotations,
         'writeinfojson': opts.writeinfojson,
         'writethumbnail': opts.writethumbnail,
+        'write_all_thumbnails': opts.write_all_thumbnails,
         'writesubtitles': opts.writesubtitles,
         'writeautomaticsub': opts.writeautomaticsub,
         'allsubtitles': opts.allsubtitles,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index e3b4b8a8a..a3b012ddb 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -638,6 +638,10 @@ def parseOpts(overrideArguments=None):
         '--write-thumbnail',
         action='store_true', dest='writethumbnail', default=False,
         help='write thumbnail image to disk')
+    thumbnail.add_option(
+        '--write-all-thumbnails',
+        action='store_true', dest='write_all_thumbnails', default=False,
+        help='write all thumbnail image formats to disk')
     thumbnail.add_option(
         '--list-thumbnails',
         action='store_true', dest='list_thumbnails', default=False,

From c14e88f0f561c5ac0a1cb9e6764fe4702bd9f7ca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:24:55 +0100
Subject: [PATCH 613/974] [YoutubeDL] Add --playlist-items option (Fixes #2662)

---
 AUTHORS                 |  1 +
 youtube_dl/YoutubeDL.py | 38 +++++++++++++++++++++++++++++++++-----
 youtube_dl/__init__.py  |  1 +
 youtube_dl/options.py   |  4 ++++
 4 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index b8bf3cb6f..8362b6d8a 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -104,3 +104,4 @@ Ondřej Caletka
 Dinesh S
 Johan K. Jensen
 Yen Chi Hsuan
+Enam Mijbah Noor
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0e73dc8ff..b7e93b8dd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -137,6 +137,7 @@ class YoutubeDL(object):
     nooverwrites:      Prevent overwriting files.
     playliststart:     Playlist item to start at.
     playlistend:       Playlist item to end at.
+    playlist_items:    Specific indices of playlist to download.
     playlistreverse:   Download playlist items in reverse order.
     matchtitle:        Download only matching titles.
     rejecttitle:       Reject downloads for matching titles.
@@ -703,24 +704,51 @@ class YoutubeDL(object):
             if playlistend == -1:
                 playlistend = None
 
+            playlistitems_str = self.params.get('playlist_items', None)
+            playlistitems = None
+            if playlistitems_str is not None:
+                def iter_playlistitems(format):
+                    for string_segment in format.split(','):
+                        if '-' in string_segment:
+                            start, end = string_segment.split('-')
+                            for item in range(int(start), int(end) + 1):
+                                yield int(item)
+                        else:
+                            yield int(string_segment)
+                playlistitems = iter_playlistitems(playlistitems_str)
+
             ie_entries = ie_result['entries']
             if isinstance(ie_entries, list):
                 n_all_entries = len(ie_entries)
-                entries = ie_entries[playliststart:playlistend]
+                if playlistitems:
+                    entries = [ie_entries[i - 1] for i in playlistitems]
+                else:
+                    entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
             elif isinstance(ie_entries, PagedList):
-                entries = ie_entries.getslice(
-                    playliststart, playlistend)
+                if playlistitems:
+                    entries = []
+                    for item in playlistitems:
+                        entries.extend(ie_entries.getslice(
+                            item - 1, item
+                        ))
+                else:
+                    entries = ie_entries.getslice(
+                        playliststart, playlistend)
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Downloading %d videos" %
                     (ie_result['extractor'], playlist, n_entries))
             else:  # iterable
-                entries = list(itertools.islice(
-                    ie_entries, playliststart, playlistend))
+                if playlistitems:
+                    entry_list = list(ie_entries)
+                    entries = [entry_list[i - 1] for i in playlistitems]
+                else:
+                    entries = list(itertools.islice(
+                        ie_entries, playliststart, playlistend))
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Downloading %d videos" %
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 04f668334..0bd7b68c3 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -333,6 +333,7 @@ def _real_main(argv=None):
         'sleep_interval': opts.sleep_interval,
         'external_downloader': opts.external_downloader,
         'list_thumbnails': opts.list_thumbnails,
+        'playlist_items': opts.playlist_items,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a3b012ddb..872835295 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -200,6 +200,10 @@ def parseOpts(overrideArguments=None):
         '--playlist-end',
         dest='playlistend', metavar='NUMBER', default=None, type=int,
         help='playlist video to end at (default is last)')
+    selection.add_option(
+        '--playlist-items',
+        dest='playlist_items', metavar='ITEM_SPEC', default=None,
+        help='playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',

From baeaeffce550b23848983cd281f173a7906cd7f9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:34:38 +0100
Subject: [PATCH 614/974] [options] Add support for infinite retries (Fixes
 #507)

---
 youtube_dl/__init__.py | 13 ++++++++-----
 youtube_dl/options.py  |  2 +-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 0bd7b68c3..09da8802d 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -143,10 +143,13 @@ def _real_main(argv=None):
             parser.error('invalid max_filesize specified')
         opts.max_filesize = numeric_limit
     if opts.retries is not None:
-        try:
-            opts.retries = int(opts.retries)
-        except (TypeError, ValueError):
-            parser.error('invalid retry count specified')
+        if opts.retries in ('inf', 'infinite'):
+            opts_retries = float('inf')
+        else:
+            try:
+                opts_retries = int(opts.retries)
+            except (TypeError, ValueError):
+                parser.error('invalid retry count specified')
     if opts.buffersize is not None:
         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
         if numeric_buffersize is None:
@@ -268,7 +271,7 @@ def _real_main(argv=None):
         'ignoreerrors': opts.ignoreerrors,
         'ratelimit': opts.ratelimit,
         'nooverwrites': opts.nooverwrites,
-        'retries': opts.retries,
+        'retries': opts_retries,
         'buffersize': opts.buffersize,
         'noresizebuffer': opts.noresizebuffer,
         'continuedl': opts.continue_dl,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 872835295..1ddbdbc78 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -377,7 +377,7 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '-R', '--retries',
         dest='retries', metavar='RETRIES', default=10,
-        help='number of retries (default is %default)')
+        help='number of retries (default is %default), or "infinite".')
     downloader.add_option(
         '--buffer-size',
         dest='buffersize', metavar='SIZE', default='1024',

From 881e6a1f5c47a65348879f817ad833081e8c5ada Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:49:44 +0100
Subject: [PATCH 615/974] Add --xattr-set-filesize option (Fixes #1348)

---
 youtube_dl/YoutubeDL.py         |  3 ++-
 youtube_dl/__init__.py          |  6 ++++++
 youtube_dl/downloader/common.py | 32 +++++++++++++++++---------------
 youtube_dl/downloader/http.py   |  8 ++++++++
 youtube_dl/options.py           |  4 ++++
 5 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b7e93b8dd..0241f7e3c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -231,7 +231,8 @@ class YoutubeDL(object):
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
-    noresizebuffer, retries, continuedl, noprogress, consoletitle
+    noresizebuffer, retries, continuedl, noprogress, consoletitle,
+    xattr_set_filesize.
 
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 09da8802d..112a8ba60 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -241,6 +241,11 @@ def _real_main(argv=None):
             'verboseOutput': opts.verbose,
             'exec_cmd': opts.exec_cmd,
         })
+    if opts.xattr_set_filesize:
+        try:
+            import xattr
+        except ImportError:
+            parser.error('setting filesize xattr requested but python-xattr is not available')
 
     ydl_opts = {
         'usenetrc': opts.usenetrc,
@@ -337,6 +342,7 @@ def _real_main(argv=None):
         'external_downloader': opts.external_downloader,
         'list_thumbnails': opts.list_thumbnails,
         'playlist_items': opts.playlist_items,
+        'xattr_set_filesize': opts.xattr_set_filesize,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index c35c42c1d..7bb3a948d 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -25,21 +25,23 @@ class FileDownloader(object):
 
     Available options:
 
-    verbose:           Print additional info to stdout.
-    quiet:             Do not print messages to stdout.
-    ratelimit:         Download speed limit, in bytes/sec.
-    retries:           Number of times to retry for HTTP error 5xx
-    buffersize:        Size of download buffer in bytes.
-    noresizebuffer:    Do not automatically resize the download buffer.
-    continuedl:        Try to continue downloads if possible.
-    noprogress:        Do not print the progress bar.
-    logtostderr:       Log messages to stderr instead of stdout.
-    consoletitle:      Display progress in console window's titlebar.
-    nopart:            Do not use temporary .part files.
-    updatetime:        Use the Last-modified header to set output file timestamps.
-    test:              Download only first bytes to test the downloader.
-    min_filesize:      Skip files smaller than this size
-    max_filesize:      Skip files larger than this size
+    verbose:            Print additional info to stdout.
+    quiet:              Do not print messages to stdout.
+    ratelimit:          Download speed limit, in bytes/sec.
+    retries:            Number of times to retry for HTTP error 5xx
+    buffersize:         Size of download buffer in bytes.
+    noresizebuffer:     Do not automatically resize the download buffer.
+    continuedl:         Try to continue downloads if possible.
+    noprogress:         Do not print the progress bar.
+    logtostderr:        Log messages to stderr instead of stdout.
+    consoletitle:       Display progress in console window's titlebar.
+    nopart:             Do not use temporary .part files.
+    updatetime:         Use the Last-modified header to set output file timestamps.
+    test:               Download only first bytes to test the downloader.
+    min_filesize:       Skip files smaller than this size
+    max_filesize:       Skip files larger than this size
+    xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
+                        (experimenatal)
 
     Subclasses of this one must re-define the real_download method.
     """
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 4db50ee90..8a1d578d5 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -157,6 +157,14 @@ class HttpFD(FileDownloader):
                 except (OSError, IOError) as err:
                     self.report_error('unable to open for writing: %s' % str(err))
                     return False
+
+                if self.params.get('xattr_set_filesize', False) and data_len is not None:
+                    try:
+                        import xattr
+                        xattr.setxattr(tmpfilename, 'user.ytdl.filesize', str(data_len))
+                    except(OSError, IOError, ImportError) as err:
+                        self.report_error('unable to set filesize xattr: %s' % str(err))
+
             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 1ddbdbc78..dbc6f5528 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -394,6 +394,10 @@ def parseOpts(overrideArguments=None):
         '--playlist-reverse',
         action='store_true',
         help='Download playlist videos in reverse order')
+    downloader.add_option(
+        '--xattr-set-filesize',
+        dest='xattr_set_filesize', action='store_true',
+        help='(experimental) set file xattribute ytdl.filesize with expected filesize')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',

From e1ccc04e9f68988df0520b7502edc6479b62378f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:56:32 +0100
Subject: [PATCH 616/974] Test rtmpdump on travis (Fixes #1601)

---
 .travis.yml                         | 3 +++
 youtube_dl/extractor/folketinget.py | 4 +---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f14014414..fb34299fc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,9 @@ python:
   - "2.7"
   - "3.3"
   - "3.4"
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -yqq rtmpdump
 script: nosetests test --verbose
 notifications:
   email:
diff --git a/youtube_dl/extractor/folketinget.py b/youtube_dl/extractor/folketinget.py
index 68e2db943..0fb29de75 100644
--- a/youtube_dl/extractor/folketinget.py
+++ b/youtube_dl/extractor/folketinget.py
@@ -16,6 +16,7 @@ class FolketingetIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ft\.dk/webtv/video/[^?#]*?\.(?P<id>[0-9]+)\.aspx'
     _TEST = {
         'url': 'http://www.ft.dk/webtv/video/20141/eru/td.1165642.aspx?as=1#player',
+        'md5': '6269e8626fa1a891bf5369b386ae996a',
         'info_dict': {
             'id': '1165642',
             'ext': 'mp4',
@@ -29,9 +30,6 @@ class FolketingetIE(InfoExtractor):
             'upload_date': '20141120',
             'duration': 3960,
         },
-        'params': {
-            'skip_download': 'rtmpdump required',
-        }
     }
 
     def _real_extract(self, url):

From 7d346331b53498b5966790677e0b41631edd8a4e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:15:47 +0100
Subject: [PATCH 617/974] [audiomack:album] Update testcase

---
 test/helper.py                    |  2 +-
 youtube_dl/extractor/audiomack.py | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index c416f388c..27a68091f 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -140,7 +140,7 @@ def expect_info_dict(self, got_dict, expected_dict):
     # Are checkable fields missing from the test case definition?
     test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
                           for key, value in got_dict.items()
-                          if value and key in ('title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
+                          if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
     missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
     if missing_keys:
         def _repr(v):
diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 8bfe50214..693ba22c6 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -88,16 +88,21 @@ class AudiomackAlbumIE(InfoExtractor):
         # Album playlist ripped from fakeshoredrive with no metadata
         {
             'url': 'http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project',
+            'info_dict': {
+                'title': 'PPP (Pistol P Project)',
+                'id': '837572',
+            },
             'playlist': [{
                 'info_dict': {
-                    'title': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
-                    'id': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
+                    'title': 'PPP (Pistol P Project) - 9. Heaven or Hell (CHIMACA) ft Zuse (prod by DJ FU)',
+                    'id': '837577',
                     'ext': 'mp3',
+                    'uploader': 'Lil Herb a.k.a. G Herbo',
                 }
             }],
             'params': {
-                'playliststart': 8,
-                'playlistend': 8,
+                'playliststart': 9,
+                'playlistend': 9,
             }
         }
     ]

From 37f4ce538ab572812465fa6d0e429bf767a030fd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:17:15 +0100
Subject: [PATCH 618/974] [smotri] Fix test case

---
 youtube_dl/extractor/smotri.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 26f361c93..e94f41362 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -102,6 +102,7 @@ class SmotriIE(InfoExtractor):
                 'uploader_id': 'mopeder',
                 'duration': 71,
                 'thumbnail': 'http://frame9.loadup.ru/d7/32/2888853.2.3.jpg',
+                'upload_date': '20150114',
             },
         },
         # swf player

From cc1237f4845e108017687d6727c3dfe6efc3cb4d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:17:38 +0100
Subject: [PATCH 619/974] [__init__] Work around flake8 false positive

---
 youtube_dl/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 112a8ba60..71d2c6f35 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -244,6 +244,7 @@ def _real_main(argv=None):
     if opts.xattr_set_filesize:
         try:
             import xattr
+            xattr  # Confuse flake8
         except ImportError:
             parser.error('setting filesize xattr requested but python-xattr is not available')
 

From 8604e882a8f99fcd632efbb94e449477d860218d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:23:21 +0100
Subject: [PATCH 620/974] [ubu] Fix test and modernize

---
 youtube_dl/extractor/ubu.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/ubu.py b/youtube_dl/extractor/ubu.py
index 0182d67ec..d50237758 100644
--- a/youtube_dl/extractor/ubu.py
+++ b/youtube_dl/extractor/ubu.py
@@ -3,50 +3,51 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    qualities,
+)
 
 
 class UbuIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?ubu\.com/film/(?P<id>[\da-z_-]+)\.html'
     _TEST = {
         'url': 'http://ubu.com/film/her_noise.html',
-        'md5': '8edd46ee8aa6b265fb5ed6cf05c36bc9',
+        'md5': '138d5652618bf0f03878978db9bef1ee',
         'info_dict': {
             'id': 'her_noise',
-            'ext': 'mp4',
+            'ext': 'm4v',
             'title': 'Her Noise - The Making Of (2007)',
             'duration': 3600,
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
             r'<title>.+?Film &amp; Video: ([^<]+)</title>', webpage, 'title')
 
         duration = int_or_none(self._html_search_regex(
-            r'Duration: (\d+) minutes', webpage, 'duration', fatal=False, default=None))
-        if duration:
-            duration *= 60
+            r'Duration: (\d+) minutes', webpage, 'duration', fatal=False),
+            invscale=60)
 
         formats = []
-
         FORMAT_REGEXES = [
-            ['sq', r"'flashvars'\s*,\s*'file=([^']+)'"],
-            ['hq', r'href="(http://ubumexico\.centro\.org\.mx/video/[^"]+)"']
+            ('sq', r"'flashvars'\s*,\s*'file=([^']+)'"),
+            ('hq', r'href="(http://ubumexico\.centro\.org\.mx/video/[^"]+)"'),
         ]
-
+        preference = qualities([fid for fid, _ in FORMAT_REGEXES])
         for format_id, format_regex in FORMAT_REGEXES:
             m = re.search(format_regex, webpage)
             if m:
                 formats.append({
                     'url': m.group(1),
                     'format_id': format_id,
+                    'preference': preference(format_id),
                 })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From ee69b99af6d00202a5ad8510604c807876d1b7b1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 06:15:51 +0100
Subject: [PATCH 621/974] [YoutubeDL] clarify hook documentation

---
 youtube_dl/YoutubeDL.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0241f7e3c..b772f87f1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -198,11 +198,12 @@ class YoutubeDL(object):
                        postprocessor.
     progress_hooks:    A list of functions that get called on download
                        progress, with a dictionary with the entries
-                       * filename: The final filename
-                       * status: One of "downloading" and "finished"
-
-                       The dict may also have some of the following entries:
+                       * status: One of "downloading" and "finished".
+                                 Check this first and ignore unknown values.
 
+                       If status is one of "downloading" or "finished", the
+                       following properties may also be present:
+                       * filename: The final filename (always present)
                        * downloaded_bytes: Bytes on disk
                        * total_bytes: Size of the whole file, None if unknown
                        * tmpfilename: The filename we're currently writing to
@@ -1251,6 +1252,7 @@ class YoutubeDL(object):
                     if self.params.get('verbose'):
                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
                     return fd.download(name, info)
+
                 if info_dict.get('requested_formats') is not None:
                     downloaded = []
                     success = True

From fe41ddbb285abccc3c4d7a3ebc1238c13ec72577 Mon Sep 17 00:00:00 2001
From: David Development <david-dev@live.de>
Date: Sun, 25 Jan 2015 11:53:53 +0100
Subject: [PATCH 622/974] refactoring - bug fixes

---
 youtube_dl/extractor/rtl2.py | 66 ++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 14b45e86e..7086d698f 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..utils import (
@@ -14,82 +13,83 @@ from ..utils import (
 
 
 class RTL2IE(InfoExtractor):
-    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
-    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))'
+    """Information Extractor for RTL2"""
+    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))/'
     _TESTS = [{
             'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
             'info_dict': {
                 'id': 'folge-203-0',
                 'ext': 'f4v',
-                'title': 'GRIP sucht den Sommerk\xf6nig',
+                'title': 'GRIP sucht den Sommerkönig',
                 'description' : 'Matthias, Det und Helge treten gegeneinander an.'
             },
             'params': {
                 # rtmp download
-                #'skip_download': True,
+                'skip_download': True,
             },
         },
         {
             'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
             'info_dict': {
                 'id': '21040-anna-erwischt-alex',
-                'ext': 'f4v',
-                'title': 'GRIP sucht den Sommerk\xf6nig',
-                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+                'ext': 'mp4',
+                'title': 'Anna erwischt Alex!',
+                'description' : 'Anna ist Alex\' Tochter bei Köln 50667.'
             },
             'params': {
                 # rtmp download
-                #'skip_download': True,
+                'skip_download': True,
             },
         },
     ]
 
     def _real_extract(self, url):
+        
+        #Some rtl2 urls have no slash at the end, so append it.
+        if not url.endswith("/"):
+            url += '/'
+        
         mobj = re.match(self._VALID_URL, url)
-        video_page_url = 'http://%s/' % mobj.group('domain')
         video_id = mobj.group('video_id')
-    
-        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
 
-        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, '%s');
-        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, '%s');
+        webpage = self._download_webpage(url, video_id)
+
+        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id not found');
+        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id not found');
 
         info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
         webpage = self._download_webpage(info_url, '')
 
-        video_info = json.loads(webpage)
+        video_info = self._download_json(info_url, video_id)
 
-        download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s');
-        title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s');
-        description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s');
-        #ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s');
+        download_url = video_info["video"]["streamurl"]
+        title = video_info["video"]["titel"]
+        description = video_info["video"]["beschreibung"]
 
         thumbnail = video_info["video"]["image"]
 
         download_url = download_url.replace("\\", "")
 
         stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s')
-
-        #print(download_url)
-        #print(stream_url)
-        #print(title)
-        #print(description)
-        #print(video_id)
         
-        formats = []
-
-        fmt = {
-            'url' : download_url,
+        #Debug output
+        #print('URL: '        + url)
+        #print('DL URL: '     + download_url)
+        #print('Stream URL: ' + stream_url)
+        #print('Title: '      + title)
+        #print('Description: '+ description)
+        #print('Video ID: '   + video_id)
+                
+        formats = [{
+                'url' : download_url,
                 #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
                 'play_path': stream_url,
                 'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
                 'page_url': url,
                 'flash_version' : "LNX 11,2,202,429",
                 'rtmp_conn' : ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"],
-                'no_resume' : 1,
-        }
-
-        formats.append(fmt)
+                'no_resume' : True,
+            }]
 
         return {
             'id': video_id,

From c9326b38b824eb8babbf2306b3bfe4bd5e683cfe Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 18:09:09 +0100
Subject: [PATCH 623/974] flake8: Ignore .git

---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 13dcd8af6..02aa56257 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,5 +2,5 @@
 universal = True
 
 [flake8]
-exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build
+exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build,.git
 ignore = E501

From 3dee7826e7133fb4a73aa0aabaee78499e49264f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 18:09:48 +0100
Subject: [PATCH 624/974] [rtl2] PEP8, simplify, make rtmp tests run (#470)

---
 youtube_dl/downloader/rtmp.py  |  13 ++--
 youtube_dl/extractor/common.py |   7 +-
 youtube_dl/extractor/rtl2.py   | 116 +++++++++++++--------------------
 3 files changed, 57 insertions(+), 79 deletions(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6f7ad588e..e06ebe826 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -105,7 +105,8 @@ class RtmpFD(FileDownloader):
         conn = info_dict.get('rtmp_conn', None)
         protocol = info_dict.get('rtmp_protocol', None)
         no_resume = info_dict.get('no_resume', False)
-        
+        continue_dl = info_dict.get('continuedl', False)
+
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
         test = self.params.get('test', False)
@@ -142,10 +143,12 @@ class RtmpFD(FileDownloader):
             basic_args += ['--conn', conn]
         if protocol is not None:
             basic_args += ['--protocol', protocol]
-        if not no_resume:
-            basic_args += ['--resume']
-        
-        args = basic_args + [[], ['--skip', '1']][not live and self.params.get('continuedl', False)]
+
+        args = basic_args
+        if not no_resume and continue_dl and not live:
+            args += ['--resume']
+        if not live and continue_dl:
+            args += ['--skip', '1']
 
         if sys.platform == 'win32' and sys.version_info < (3, 0):
             # Windows subprocess module does not actually support Unicode
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7b7a832dc..388c55e99 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -114,8 +114,11 @@ class InfoExtractor(object):
                     * http_post_data  Additional data to send with a POST
                                  request.
                     * stretched_ratio  If given and not 1, indicates that the
-                                       video's pixels are not square.
-                                       width : height ratio as float.
+                                 video's pixels are not square.
+                                 width : height ratio as float.
+                    * no_resume  The server does not support resuming the
+                                 (HTTP or RTMP) download. Boolean.
+
     url:            Final video URL.
     ext:            Video filename extension.
     format:         The video format, defaults to ext (used for --get-format)
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 7086d698f..72cd80498 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -1,100 +1,72 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    clean_html,
-    unified_strdate,
-    int_or_none,
-)
 
 
 class RTL2IE(InfoExtractor):
-    """Information Extractor for RTL2"""
-    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))/'
+    _VALID_URL = r'http?://(?:www\.)?rtl2\.de/[^?#]*?/(?P<id>[^?#/]*?)(?:$|/(?:$|[?#]))'
     _TESTS = [{
-            'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
-            'info_dict': {
-                'id': 'folge-203-0',
-                'ext': 'f4v',
-                'title': 'GRIP sucht den Sommerkönig',
-                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
+        'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
+        'md5': 'bfcc179030535b08dc2b36b469b5adc7',
+        'info_dict': {
+            'id': 'folge-203-0',
+            'ext': 'f4v',
+            'title': 'GRIP sucht den Sommerkönig',
+            'description': 'Matthias, Det und Helge treten gegeneinander an.'
         },
-        {
-            'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
-            'info_dict': {
-                'id': '21040-anna-erwischt-alex',
-                'ext': 'mp4',
-                'title': 'Anna erwischt Alex!',
-                'description' : 'Anna ist Alex\' Tochter bei Köln 50667.'
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
+    }, {
+        'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
+        'md5': 'ffcd517d2805b57ce11a58a2980c2b02',
+        'info_dict': {
+            'id': '21040-anna-erwischt-alex',
+            'ext': 'mp4',
+            'title': 'Anna erwischt Alex!',
+            'description': 'Anna ist Alex\' Tochter bei Köln 50667.'
         },
-    ]
+    }]
 
     def _real_extract(self, url):
-        
-        #Some rtl2 urls have no slash at the end, so append it.
-        if not url.endswith("/"):
+        # Some rtl2 urls have no slash at the end, so append it.
+        if not url.endswith('/'):
             url += '/'
-        
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
 
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id not found');
-        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id not found');
-
+        vico_id = self._html_search_regex(
+            r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id')
+        vivi_id = self._html_search_regex(
+            r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id')
         info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
         webpage = self._download_webpage(info_url, '')
 
-        video_info = self._download_json(info_url, video_id)
+        info = self._download_json(info_url, video_id)
+        video_info = info['video']
+        title = video_info['titel']
+        description = video_info.get('beschreibung')
+        thumbnail = video_info.get('image')
 
-        download_url = video_info["video"]["streamurl"]
-        title = video_info["video"]["titel"]
-        description = video_info["video"]["beschreibung"]
+        download_url = video_info['streamurl']
+        download_url = download_url.replace('\\', '')
+        stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, 'stream URL')
+        rtmp_conn = ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"]
 
-        thumbnail = video_info["video"]["image"]
-
-        download_url = download_url.replace("\\", "")
-
-        stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s')
-        
-        #Debug output
-        #print('URL: '        + url)
-        #print('DL URL: '     + download_url)
-        #print('Stream URL: ' + stream_url)
-        #print('Title: '      + title)
-        #print('Description: '+ description)
-        #print('Video ID: '   + video_id)
-                
         formats = [{
-                'url' : download_url,
-                #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
-                'play_path': stream_url,
-                'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
-                'page_url': url,
-                'flash_version' : "LNX 11,2,202,429",
-                'rtmp_conn' : ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"],
-                'no_resume' : True,
-            }]
+            'url': download_url,
+            'play_path': stream_url,
+            'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
+            'page_url': url,
+            'flash_version': 'LNX 11,2,202,429',
+            'rtmp_conn': rtmp_conn,
+            'no_resume': True,
+        }]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'thumbnail' : thumbnail,
-            'description' : description,
+            'thumbnail': thumbnail,
+            'description': description,
             'formats': formats,
         }

From b1b0b1ca3030a2b04dd94ec28d5ece3fda88a282 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 18:14:59 +0100
Subject: [PATCH 625/974] [generic] Improve description testcase in rss test

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a028c4ed4..ad16b8330 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -362,7 +362,7 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
                 'title': 'Zero Punctuation',
-                'description': 're:'
+                'description': 're:.*groundbreaking video review series.*'
             },
             'playlist_mincount': 11,
         },

From 6d2749aac407df1e039f5b61a294991c1e810cff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 25 Jan 2015 18:56:04 +0100
Subject: [PATCH 626/974] [drtv] Prefer the version without spoken subtitles
 (fixes #4779)

For example for http://www.dr.dk/tv/se/moderne-klassikere/moderne-klassikere-one-republic-apologize#!/, there's a version where everytime someone speaks in English a computer voice translates it.
---
 youtube_dl/extractor/drtv.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index c44adb109..510ef04b0 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -48,14 +48,20 @@ class DRTVIE(SubtitlesInfoExtractor):
             elif asset['Kind'] == 'VideoResource':
                 duration = asset['DurationInMilliseconds'] / 1000.0
                 restricted_to_denmark = asset['RestrictedToDenmark']
+                spoken_subtitles = asset['Target'] == 'SpokenSubtitles'
                 for link in asset['Links']:
                     target = link['Target']
                     uri = link['Uri']
+                    format_id = target
+                    preference = -1 if target == 'HDS' else -2
+                    if spoken_subtitles:
+                        preference -= 2
+                        format_id += '-spoken-subtitles'
                     formats.append({
                         'url': uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43' if target == 'HDS' else uri,
-                        'format_id': target,
+                        'format_id': format_id,
                         'ext': link['FileFormat'],
-                        'preference': -1 if target == 'HDS' else -2,
+                        'preference': preference,
                     })
                 subtitles_list = asset.get('SubtitlesList')
                 if isinstance(subtitles_list, list):

From 96a53167fa64293506f446d0c2bf3e0db6c8df31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 00:32:31 +0600
Subject: [PATCH 627/974] [common] Generalize URLs' HTTP errors pre-testing

---
 youtube_dl/extractor/common.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 388c55e99..478232682 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -14,6 +14,7 @@ import xml.etree.ElementTree
 
 from ..compat import (
     compat_cookiejar,
+    compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
@@ -26,6 +27,7 @@ from ..utils import (
     compiled_regex_type,
     ExtractorError,
     float_or_none,
+    HEADRequest,
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,
@@ -716,6 +718,27 @@ class InfoExtractor(object):
             )
         formats.sort(key=_formats_key)
 
+    def _check_formats(self, formats, video_id):
+        if formats:
+            formats[:] = filter(
+                lambda f: self._is_valid_url(
+                    f['url'], video_id,
+                    item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
+                formats)
+
+    def _is_valid_url(self, url, video_id, item='video'):
+        try:
+            self._request_webpage(
+                HEADRequest(url), video_id,
+                'Checking %s URL' % item)
+            return True
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                self.report_warning(
+                    '%s URL is invalid, skipping' % item, video_id)
+                return False
+            raise
+
     def http_scheme(self):
         """ Either "http:" or "https:", depending on the user's preferences """
         return (

From a57e8ce6580202c179c38a15abc31f84ca471521 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 00:33:42 +0600
Subject: [PATCH 628/974] [lynda] Pre-test video URLs for HTTP errors (Closes
 #2185, closes #4782)

---
 youtube_dl/extractor/lynda.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 26e84970d..762cefa34 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -85,6 +85,7 @@ class LyndaIE(SubtitlesInfoExtractor):
                 } for format_id, video_url in prioritized_streams['0'].items()
             ])
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         if self._downloader.params.get('listsubtitles', False):

From d862a4f94fac46e5c6de790a9b66d78d463d666d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 00:34:31 +0600
Subject: [PATCH 629/974] [spiegel] Use generalized formats pre-testing

---
 youtube_dl/extractor/spiegel.py | 20 ++------------------
 1 file changed, 2 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index f345883c7..b868241d5 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -4,14 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
-    compat_HTTPError,
-)
-from ..utils import (
-    HEADRequest,
-    ExtractorError,
-)
+from ..compat import compat_urlparse
 from .spiegeltv import SpiegeltvIE
 
 
@@ -72,16 +65,6 @@ class SpiegelIE(InfoExtractor):
             if n.tag.startswith('type') and n.tag != 'type6':
                 format_id = n.tag.rpartition('type')[2]
                 video_url = base_url + n.find('./filename').text
-                # Test video URLs beforehand as some of them are invalid
-                try:
-                    self._request_webpage(
-                        HEADRequest(video_url), video_id,
-                        'Checking %s video URL' % format_id)
-                except ExtractorError as e:
-                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
-                        self.report_warning(
-                            '%s video URL is invalid, skipping' % format_id, video_id)
-                        continue
                 formats.append({
                     'format_id': format_id,
                     'url': video_url,
@@ -94,6 +77,7 @@ class SpiegelIE(InfoExtractor):
                 })
         duration = float(idoc[0].findall('./duration')[0].text)
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         return {

From 80a49d3d7bcd235ba15bd491cc62a0345c9abce1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 02:08:29 +0600
Subject: [PATCH 630/974] Credit @David-Development for rtl2 (#4780)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 8362b6d8a..1596a7548 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -105,3 +105,4 @@ Dinesh S
 Johan K. Jensen
 Yen Chi Hsuan
 Enam Mijbah Noor
+David Luhmer

From 2b1bd292ae72fdea4ab731f93777a99b3729b31d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 21:40:43 +0100
Subject: [PATCH 631/974] release 2015.01.25

---
 README.md             | 23 +++++++++++++++++++++--
 youtube_dl/version.py |  2 +-
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 36b87444e..0580ab2a3 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,14 @@ which means you can modify it, redistribute it or use it however you like.
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
+    --playlist-items ITEM_SPEC       playlist video items to download. Specify
+                                     indices of the videos in the playlist
+                                     seperated by commas like: "--playlist-items
+                                     1,2,5,8" if you want to download videos
+                                     indexed 1, 2, 5, 8 in the playlist. You can
+                                     specify range: "--playlist-items
+                                     1-3,7,10-13", it will download the videos
+                                     at index 1, 2, 3, 7, 10, 11, 12 and 13.
     --match-title REGEX              download only matching titles (regex or
                                      caseless sub-string)
     --reject-title REGEX             skip download for matching titles (regex or
@@ -124,7 +132,8 @@ which means you can modify it, redistribute it or use it however you like.
 ## Download Options:
     -r, --rate-limit LIMIT           maximum download rate in bytes per second
                                      (e.g. 50K or 4.2M)
-    -R, --retries RETRIES            number of retries (default is 10)
+    -R, --retries RETRIES            number of retries (default is 10), or
+                                     "infinite".
     --buffer-size SIZE               size of download buffer (e.g. 1024 or 16K)
                                      (default is 1024)
     --no-resize-buffer               do not automatically adjust the buffer
@@ -132,6 +141,11 @@ which means you can modify it, redistribute it or use it however you like.
                                      automatically resized from an initial value
                                      of SIZE.
     --playlist-reverse               Download playlist videos in reverse order
+    --xattr-set-filesize             (experimental) set file xattribute
+                                     ytdl.filesize with expected filesize
+    --external-downloader COMMAND    (experimental) Use the specified external
+                                     downloader. Currently supports
+                                     aria2c,curl,wget
 
 ## Filesystem Options:
     -a, --batch-file FILE            file containing URLs to download ('-' for
@@ -191,7 +205,6 @@ which means you can modify it, redistribute it or use it however you like.
     --write-info-json                write video metadata to a .info.json file
     --write-annotations              write video annotations to a .annotation
                                      file
-    --write-thumbnail                write thumbnail image to disk
     --load-info FILE                 json file containing the video information
                                      (created with the "--write-json" option)
     --cookies FILE                   file to read cookies from and dump cookie
@@ -206,6 +219,12 @@ which means you can modify it, redistribute it or use it however you like.
     --no-cache-dir                   Disable filesystem caching
     --rm-cache-dir                   Delete all filesystem cache files
 
+## Thumbnail images:
+    --write-thumbnail                write thumbnail image to disk
+    --write-all-thumbnails           write all thumbnail image formats to disk
+    --list-thumbnails                Simulate and list all available thumbnail
+                                     formats
+
 ## Verbosity / Simulation Options:
     -q, --quiet                      activates quiet mode
     --no-warnings                    Ignore warnings
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 35f3e1b6b..0d59cddd7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.4'
+__version__ = '2015.01.25'

From e72c7e41238681338c92427f6731c74947645308 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 26 Jan 2015 12:01:43 +0100
Subject: [PATCH 632/974] [YoutubeDL] Always set the '_filename' field in the
 info_dict (reported in #4053)

It's also useful when you use the '--write-info-json' option.
---
 youtube_dl/YoutubeDL.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b772f87f1..ae62432c6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1130,7 +1130,7 @@ class YoutubeDL(object):
 
         self._num_downloads += 1
 
-        filename = self.prepare_filename(info_dict)
+        info_dict['_filename'] = filename = self.prepare_filename(info_dict)
 
         # Forced printings
         if self.params.get('forcetitle', False):
@@ -1155,10 +1155,7 @@ class YoutubeDL(object):
         if self.params.get('forceformat', False):
             self.to_stdout(info_dict['format'])
         if self.params.get('forcejson', False):
-            info_dict['_filename'] = filename
             self.to_stdout(json.dumps(info_dict))
-        if self.params.get('dump_single_json', False):
-            info_dict['_filename'] = filename
 
         # Do nothing else if in simulate mode
         if self.params.get('simulate', False):

From 9f0df77ab13ca19bc074b501296bb6a494193cd1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 26 Jan 2015 14:36:22 +0100
Subject: [PATCH 633/974] [YoutubeDL] Allow format filtering by fps

---
 youtube_dl/YoutubeDL.py | 2 +-
 youtube_dl/options.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ae62432c6..b5dd77e3f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -820,7 +820,7 @@ class YoutubeDL(object):
             '!=': operator.ne,
         }
         operator_rex = re.compile(r'''(?x)\s*\[
-            (?P<key>width|height|tbr|abr|vbr|filesize)
+            (?P<key>width|height|tbr|abr|vbr|filesize|fps)
             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
             \]$
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index dbc6f5528..5e4a1ad9b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -297,7 +297,7 @@ def parseOpts(overrideArguments=None):
             ' You can filter the video results by putting a condition in'
             ' brackets, as in -f "best[height=720]"'
             ' (or -f "[filesize>10M]"). '
-            ' This works for filesize, height, width, tbr, abr, and vbr'
+            ' This works for filesize, height, width, tbr, abr, vbr, and fps'
             ' and the comparisons <, <=, >, >=, =, != .'
             ' Formats for which the value is not known are excluded unless you'
             ' put a question mark (?) after the operator.'

From 6ca85be6f8697dc8cb0378854c2c6cdc154593d4 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Wed, 28 May 2014 18:19:23 +0200
Subject: [PATCH 634/974] Filter DRM protected media in f4m downloader

---
 youtube_dl/downloader/f4m.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index c68b2c303..29de7630d 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -230,6 +230,23 @@ class F4mFD(FileDownloader):
     A downloader for f4m manifests or AdobeHDS.
     """
 
+    def _get_unencrypted_media(self, doc):
+        media=doc.findall(_add_ns('media'))
+        if not media:
+            self.report_error('No media found')
+        for e in (doc.findall(_add_ns('drmAdditionalHeader')) +
+                  doc.findall(_add_ns('drmAdditionalHeaderSet'))):
+            # If id attribute is missing it's valid for all media nodes
+            # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
+            if not 'id' in e.attrib:
+                self.report_error('Media is DRM protected')
+        media = list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib and
+                                      'drmAdditionalHeaderSetId' not in e.attrib,
+                            media))
+        if not media:
+            self.report_error('Media is DRM protected')
+        return media
+
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']
         requested_bitrate = info_dict.get('tbr')
@@ -248,7 +265,8 @@ class F4mFD(FileDownloader):
         )
 
         doc = etree.fromstring(manifest)
-        formats = [(int(f.attrib.get('bitrate', -1)), f) for f in doc.findall(_add_ns('media'))]
+        formats = [(int(f.attrib.get('bitrate', -1)), f)
+                   for f in self._get_unencrypted_media(doc)]
         if requested_bitrate is None:
             # get the best format
             formats = sorted(formats, key=lambda f: f[0])

From 2103d038b3dac6f489d13282dcd2a6f4b74975c3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 27 Jan 2015 15:38:54 +0200
Subject: [PATCH 635/974] [lnkgo] Adapt to website changes

---
 youtube_dl/extractor/lnkgo.py | 61 ++++++++++++++---------------------
 1 file changed, 25 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/lnkgo.py b/youtube_dl/extractor/lnkgo.py
index a8e357859..fd23b0b43 100644
--- a/youtube_dl/extractor/lnkgo.py
+++ b/youtube_dl/extractor/lnkgo.py
@@ -6,13 +6,12 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    js_to_json,
     unified_strdate,
 )
 
 
 class LnkGoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?lnkgo\.alfa\.lt/visi\-video/(?P<show>[^/]+)/ziurek\-(?P<display_id>[A-Za-z0-9\-]+)'
+    _VALID_URL = r'https?://(?:www\.)?lnkgo\.alfa\.lt/visi-video/(?P<show>[^/]+)/ziurek-(?P<id>[A-Za-z0-9-]+)'
     _TESTS = [{
         'url': 'http://lnkgo.alfa.lt/visi-video/yra-kaip-yra/ziurek-yra-kaip-yra-162',
         'info_dict': {
@@ -51,8 +50,7 @@ class LnkGoIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = self._match_id(url)
 
         webpage = self._download_webpage(
             url, display_id, 'Downloading player webpage')
@@ -61,6 +59,8 @@ class LnkGoIE(InfoExtractor):
             r'data-ep="([^"]+)"', webpage, 'video ID')
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)
+        upload_date = unified_strdate(self._search_regex(
+            r'class="[^"]*meta-item[^"]*air-time[^"]*">.*?<strong>([^<]+)</strong>', webpage, 'upload date', fatal=False))
 
         thumbnail_w = int_or_none(
             self._og_search_property('image:width', webpage, 'thumbnail width', fatal=False))
@@ -75,39 +75,28 @@ class LnkGoIE(InfoExtractor):
                 'height': thumbnail_h,
             })
 
-        upload_date = unified_strdate(self._search_regex(
-            r'class="meta-item\sair-time">.*?<strong>([^<]+)</strong>', webpage, 'upload date', fatal=False))
-        duration = int_or_none(self._search_regex(
-            r'VideoDuration = "([^"]+)"', webpage, 'duration', fatal=False))
+        config = self._parse_json(self._search_regex(
+            r'episodePlayer\((\{.*?\}),\s*\{', webpage, 'sources'), video_id)
 
-        pg_rating = self._search_regex(
-            r'pgrating="([^"]+)"', webpage, 'PG rating', fatal=False, default='')
-        age_limit = self._AGE_LIMITS.get(pg_rating.upper(), 0)
+        if config.get('pGeo'):
+            self.report_warning(
+                'This content might not be available in your country due to copyright reasons')
 
-        sources_js = self._search_regex(
-            r'(?s)sources:\s(\[.*?\]),', webpage, 'sources')
-        sources = self._parse_json(
-            sources_js, video_id, transform_source=js_to_json)
+        formats = [{
+            'format_id': 'hls',
+            'ext': 'mp4',
+            'url': config['EpisodeVideoLink_HLS'],
+        }]
 
-        formats = []
-        for source in sources:
-            if source.get('provider') == 'rtmp':
-                m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<play_path>.+)$', source['file'])
-                if not m:
-                    continue
-                formats.append({
-                    'format_id': 'rtmp',
-                    'ext': 'flv',
-                    'url': m.group('url'),
-                    'play_path': m.group('play_path'),
-                    'page_url': url,
-                })
-            elif source.get('file').endswith('.m3u8'):
-                formats.append({
-                    'format_id': 'hls',
-                    'ext': source.get('type', 'mp4'),
-                    'url': source['file'],
-                })
+        m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<play_path>.+)$', config['EpisodeVideoLink'])
+        if m:
+            formats.append({
+                'format_id': 'rtmp',
+                'ext': 'flv',
+                'url': m.group('url'),
+                'play_path': m.group('play_path'),
+                'page_url': url,
+            })
 
         self._sort_formats(formats)
 
@@ -117,8 +106,8 @@ class LnkGoIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'thumbnails': [thumbnail],
-            'duration': duration,
+            'duration': int_or_none(config.get('VideoTime')),
             'description': description,
-            'age_limit': age_limit,
+            'age_limit': self._AGE_LIMITS.get(config.get('PGRating'), 0),
             'upload_date': upload_date,
         }

From fe7710cbccdade0b66ef48a7e2eedad71a0702cc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 27 Jan 2015 23:55:22 +0800
Subject: [PATCH 636/974] [xuite] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/xuite.py    | 147 +++++++++++++++++++++++++++++++
 2 files changed, 148 insertions(+)
 create mode 100644 youtube_dl/extractor/xuite.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..09070daa4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -546,6 +546,7 @@ from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
+from .xuite import XuiteIE
 from .xxxymovies import XXXYMoviesIE
 from .yahoo import (
     YahooIE,
diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
new file mode 100644
index 000000000..943757c27
--- /dev/null
+++ b/youtube_dl/extractor/xuite.py
@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import base64
+from .common import InfoExtractor
+from ..compat import (
+    compat_urlparse,
+    compat_urllib_parse_unquote,
+    compat_parse_qs
+)
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+    parse_duration
+)
+
+# ref: http://stackoverflow.com/questions/475074/regex-to-parse-or-validate-base64-data
+REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
+
+CST_ZONE = +8  # China Standard Time
+
+
+class XuiteIE(InfoExtractor):
+    _VALID_URL = r'http://vlog.xuite.net/play/(?P<id>%s)(/.*)?' % REGEX_BASE64
+    _TESTS = [{
+        # Audio
+        'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',
+        'md5': '63a42c705772aa53fd4c1a0027f86adf',
+        'info_dict': {
+            'id': 'RGkzc1ZULTM4NjA5MTQuZmx2',
+            'ext': 'mp3',
+            'upload_date': '20110902',
+            'uploader_id': '15973816',
+            'uploader': '阿能',
+            'timestamp': 1314932940,
+            'title': '孤單南半球-歐德陽'
+        }
+    }, {
+        # Audio with alternative form of url
+        'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
+        'md5': 'c91645f324de53d82ebb80930e1a73d2',
+        'info_dict': {
+            'id': 'S1dDUjdyLTMyOTc3NjcuZmx2',
+            'ext': 'mp3',
+            'upload_date': '20101226',
+            'uploader_id': '10102699',
+            'uploader': '蠍',
+            'timestamp': 1293367080,
+            'title': '孫燕姿-眼淚成詩',
+        }
+    }, {
+        # Video with only one format
+        'url': 'http://vlog.xuite.net/play/TkRZNjhULTM0NDE2MjkuZmx2',
+        'md5': 'c45737fc8ac5dc8ac2f92ecbcecf505e',
+        'info_dict': {
+            'id': 'TkRZNjhULTM0NDE2MjkuZmx2',
+            'ext': 'mp4',
+            'upload_date': '20110306',
+            'uploader_id': '10400126',
+            'uploader': 'Valen',
+            'timestamp': 1299383640,
+            'title': '孫燕姿 - 眼淚成詩',
+        }
+    }, {
+        # Video with two formats
+        'url': 'http://vlog.xuite.net/play/bWo1N1pLLTIxMzAxMTcwLmZsdg==',
+        'md5': '1166e0f461efe55b62e26a2d2a68e6de',
+        'info_dict': {
+            'id': 'bWo1N1pLLTIxMzAxMTcwLmZsdg==',
+            'ext': 'mp4',
+            'upload_date': '20150117',
+            'uploader_id': '242127761',
+            'uploader': '我只是想認真點',
+            'timestamp': 1421481240,
+            'title': '暗殺教室 02',
+        }
+    }]
+
+    def _flv_config(self, media_id):
+        base64_media_id = base64.b64encode(media_id.encode('utf-8')).decode('utf-8')
+        flv_config_url = 'http://vlog.xuite.net/flash/player?media=' + base64_media_id
+        flv_config = self._download_xml(flv_config_url, 'flv config')
+
+        prop_dict = {}
+        for prop in flv_config.findall('./property'):
+            prop_id = base64.b64decode(prop.attrib['id']).decode('utf-8')
+
+            if not prop.text:
+                continue  # CDATA may be empty in flv config
+
+            encoded_content = base64.b64decode(prop.text).decode('utf-8')
+            prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content)
+
+        return prop_dict
+
+    def _type_string(self, media_url):
+        query_string = compat_urlparse.urlparse(media_url).query
+        type_string = compat_parse_qs(query_string)['q'][0]
+        return type_string
+
+    def _guess_ext(self, media_url):
+        type_string = self._type_string(media_url)
+        if type_string == 'mp3':
+            return 'mp3'
+        elif type_string == '360' or type_string == '720':
+            return 'mp4'
+        else:
+            raise ExtractorError('Unknown type string %s' % type_string)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        page = self._download_webpage(url, video_id)
+        media_id = self._html_search_regex(r'data-mediaid="(\d+)"', page, 'media id')
+        flv_config = self._flv_config(media_id)
+
+        timestamp_local = parse_iso8601(flv_config['publish_datetime'], ' ')
+        timestamp_gmt = timestamp_local - CST_ZONE * 3600
+
+        ret_attrs = {
+            'id': video_id,
+            'title': flv_config['title'],
+            'thumbnail': flv_config['thumb'],
+            'uploader': flv_config['author_name'],
+            'timestamp': timestamp_gmt,
+            'uploader_id': flv_config['author_id'],
+            'duration': parse_duration(flv_config['duration']),
+            'categories': [flv_config['category']]
+        }
+
+        if 'hq_src' in flv_config:
+            src = flv_config['src']
+            src_hq = flv_config['hq_src']
+            ret_attrs['formats'] = [{
+                'url': src,
+                'ext': self._guess_ext(src),
+                'format_id': self._type_string(src)
+            }, {
+                'url': src_hq,
+                'ext': self._guess_ext(src_hq),
+                'format_id': self._type_string(src_hq)
+            }]
+        else:
+            ret_attrs['url'] = flv_config['src']
+            ret_attrs['ext'] = self._guess_ext(flv_config['src'])
+
+        return ret_attrs

From 6348ad12a057cc1c454488ce7c37070cf39a8f06 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 28 Jan 2015 00:13:40 +0800
Subject: [PATCH 637/974] [xuite] Add height information for the two formats

---
 youtube_dl/extractor/xuite.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 943757c27..aef9a14a3 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -129,17 +129,17 @@ class XuiteIE(InfoExtractor):
         }
 
         if 'hq_src' in flv_config:
-            src = flv_config['src']
-            src_hq = flv_config['hq_src']
-            ret_attrs['formats'] = [{
-                'url': src,
-                'ext': self._guess_ext(src),
-                'format_id': self._type_string(src)
-            }, {
-                'url': src_hq,
-                'ext': self._guess_ext(src_hq),
-                'format_id': self._type_string(src_hq)
-            }]
+            urls = [flv_config['src'], flv_config['hq_src']]
+
+            ret_attrs['formats'] = []
+
+            for url in urls:
+                ret_attrs['formats'].append({
+                    'url': url,
+                    'ext': self._guess_ext(url),
+                    'format_id': self._type_string(url),
+                    'height': int(self._type_string(url))
+                })
         else:
             ret_attrs['url'] = flv_config['src']
             ret_attrs['ext'] = self._guess_ext(flv_config['src'])

From 3a0d2f520a0f95c2f87b1c95049135b10206f97f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Jan 2015 22:38:28 +0600
Subject: [PATCH 638/974] [YoutubeDL] Temporary fix for subprocess encoding
 issues on python2 @ Windows (Closes #4787)

For now filenames will be encoded with preferrefencoding before written to disk
---
 youtube_dl/YoutubeDL.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b5dd77e3f..7f054cdff 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -543,6 +543,11 @@ class YoutubeDL(object):
             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
+            # Temporary fix for #4787
+            # 'Treat' all problem characters by passing filename through preferredencoding
+            # to workaround encoding issues with subprocess on python2 @ Windows
+            if sys.version_info < (3, 0) and sys.platform == 'win32':
+                filename = encodeFilename(filename, True).decode(preferredencoding())
             return filename
         except ValueError as err:
             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')

From a28383834b07787b22a837bd6028bf24bea7ec94 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 28 Jan 2015 01:30:14 +0800
Subject: [PATCH 639/974] [xuite] Update tests

---
 youtube_dl/extractor/xuite.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index aef9a14a3..8466d4bc5 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -33,7 +33,10 @@ class XuiteIE(InfoExtractor):
             'uploader_id': '15973816',
             'uploader': '阿能',
             'timestamp': 1314932940,
-            'title': '孤單南半球-歐德陽'
+            'title': '孤單南半球-歐德陽',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['個人短片'],
+            'duration': 247.246
         }
     }, {
         # Audio with alternative form of url
@@ -47,6 +50,9 @@ class XuiteIE(InfoExtractor):
             'uploader': '蠍',
             'timestamp': 1293367080,
             'title': '孫燕姿-眼淚成詩',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['個人短片'],
+            'duration': 223.19
         }
     }, {
         # Video with only one format
@@ -60,6 +66,9 @@ class XuiteIE(InfoExtractor):
             'uploader': 'Valen',
             'timestamp': 1299383640,
             'title': '孫燕姿 - 眼淚成詩',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['影視娛樂'],
+            'duration': 217.399
         }
     }, {
         # Video with two formats
@@ -73,6 +82,9 @@ class XuiteIE(InfoExtractor):
             'uploader': '我只是想認真點',
             'timestamp': 1421481240,
             'title': '暗殺教室 02',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['電玩動漫'],
+            'duration': 1384.907
         }
     }]
 

From 796df3c631dcf28f7ebdf256e57fcfa3b6463abe Mon Sep 17 00:00:00 2001
From: Shaya G <hyshai29@gmail.com>
Date: Wed, 28 Jan 2015 00:08:19 -0500
Subject: [PATCH 640/974] fixed viddler support - needed a Referer header; also
 added a viddler generic extractor

---
 youtube_dl/extractor/generic.py | 20 ++++++++++++++++++
 youtube_dl/extractor/viddler.py | 36 +++++++++++++++++++++++++++++----
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ad16b8330..a937a1e09 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -498,6 +498,19 @@ class GenericIE(InfoExtractor):
                 'uploader': 'www.abc.net.au',
                 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
             }
+        },
+        # embedded viddler video
+        {
+            'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
+            'info_dict': {
+                'id': '4d03aad9',
+                'ext': 'mp4',
+                'uploader': 'deadspin',
+                'title': 'WALL-TO-GORTAT',
+                'timestamp': 1422285291,
+                'upload_date': '20150126',
+            },
+            'add_ie': ['Viddler'],
         }
     ]
 
@@ -860,6 +873,13 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
+        # Look for embedded Viddler player
+        mobj = (re.search(r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/embed/.+?)\1', webpage) or
+                re.search(r'<param[^>]+?value=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/player/.+?)\1', webpage))
+
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for Ooyala videos
         mobj = (re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO.Player.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 0faa729c6..bacbad50a 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -5,11 +5,14 @@ from ..utils import (
     float_or_none,
     int_or_none,
 )
+from ..compat import (
+   compat_urllib_request
+)
 
 
 class ViddlerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         "url": "http://www.viddler.com/v/43903784",
         'md5': 'ae43ad7cb59431ce043f0ff7fa13cbf4',
         'info_dict': {
@@ -25,7 +28,30 @@ class ViddlerIE(InfoExtractor):
             'view_count': int,
             'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
         }
-    }
+    }, {
+        "url": "http://www.viddler.com/v/4d03aad9/",
+        "file": "4d03aad9.mp4",
+        "md5": "faa71fbf70c0bee7ab93076fd007f4b0",
+        "info_dict": {
+            'upload_date': '20150126',
+            'uploader': 'deadspin',
+            'id': '4d03aad9',
+            'timestamp': 1422285291,
+            'title': 'WALL-TO-GORTAT',
+        }
+    }, {
+        "url": "http://www.viddler.com/player/221ebbbd/0/",
+        "file": "221ebbbd.mp4",
+        "md5": "0defa2bd0ea613d14a6e9bd1db6be326",
+        "info_dict": {
+            'upload_date': '20140929',
+            'uploader': 'BCLETeens',
+            'id': '221ebbbd',
+            'timestamp': 1411997190,
+            'title': 'LETeens-Grammar-snack-third-conditional',
+            'description': ' '
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -33,7 +59,9 @@ class ViddlerIE(InfoExtractor):
         json_url = (
             'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
             video_id)
-        data = self._download_json(json_url, video_id)['video']
+        headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
+        request = compat_urllib_request.Request(json_url, None, headers)
+        data = self._download_json(request, video_id)['video']
 
         formats = []
         for filed in data['files']:
@@ -53,7 +81,7 @@ class ViddlerIE(InfoExtractor):
 
             if filed.get('cdn_url'):
                 f = f.copy()
-                f['url'] = self._proto_relative_url(filed['cdn_url'])
+                f['url'] = self._proto_relative_url(filed['cdn_url'], 'http:')
                 f['format_id'] = filed['profile_id'] + '-cdn'
                 f['source_preference'] = 1
                 formats.append(f)

From 0865f397ae6c875ec4194093af5c8f53e75f2285 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Wed, 28 Jan 2015 08:21:04 +0100
Subject: [PATCH 641/974] added extractor for dctp.tv

---
 youtube_dl/downloader/rtmp.py    |  3 +++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dctp.py     | 41 ++++++++++++++++++++++++++++++++
 3 files changed, 45 insertions(+)
 create mode 100644 youtube_dl/extractor/dctp.py

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index e06ebe826..6cb1bfc50 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -106,6 +106,7 @@ class RtmpFD(FileDownloader):
         protocol = info_dict.get('rtmp_protocol', None)
         no_resume = info_dict.get('no_resume', False)
         continue_dl = info_dict.get('continuedl', False)
+        real_time = info_dict.get('real_time', False)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
@@ -143,6 +144,8 @@ class RtmpFD(FileDownloader):
             basic_args += ['--conn', conn]
         if protocol is not None:
             basic_args += ['--protocol', protocol]
+        if real_time:
+            basic_args += ['--realtime']
 
         args = basic_args
         if not no_resume and continue_dl and not live:
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..873ae69d3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -89,6 +89,7 @@ from .dailymotion import (
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
+from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
 from .dotsub import DotsubIE
diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
new file mode 100644
index 000000000..5382255f0
--- /dev/null
+++ b/youtube_dl/extractor/dctp.py
@@ -0,0 +1,41 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+class DctpTvIE(InfoExtractor):
+    _VALID_URL = r'^http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        base_url = 'http://dctp-ivms2-restapi.s3.amazonaws.com/'
+        version_json = self._download_json(base_url + 'version.json', video_id)
+        version = version_json['version_name']
+        info_json = self._download_json(
+            '{}{}/restapi/slugs/{}.json'.format(base_url, version, video_id), video_id)
+        object_id = info_json['object_id']
+        meta_json = self._download_json(
+            '{}{}/restapi/media/{}.json'.format(base_url, version, object_id), video_id)
+        uuid = meta_json['uuid']
+        title = meta_json['title']
+        wide = meta_json['is_wide']
+        if wide:
+            ratio = '16x9'
+        else:
+            ratio = '4x3'
+        play_path = 'mp4:{}_dctp_0500_{}.m4v'.format(uuid, ratio)
+
+        servers_json = self._download_json('http://www.dctp.tv/streaming_servers/', video_id)
+        url = servers_json[0]['endpoint']
+
+        return {
+            'id': video_id,
+            'title': title,
+            'format': 'rtmp',
+            'url': url,
+            'play_path': play_path,
+            'real_time': True,
+            'ext': 'flv'
+        }
+
+

From 48a1e5141ad9f6b5c4ce8a355dcd7bf99c80e333 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Wed, 28 Jan 2015 08:59:58 +0100
Subject: [PATCH 642/974] added test for dctp

---
 youtube_dl/extractor/dctp.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 5382255f0..9b687ef43 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -3,8 +3,16 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
+
 class DctpTvIE(InfoExtractor):
     _VALID_URL = r'^http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+    _TEST = {
+        'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
+        'info_dict': {
+            'id': 'videoinstallation-fuer-eine-kaufhausfassade',
+            'ext': 'flv',
+            'title': 'Videoinstallation für eine Kaufhausfassade'}
+        }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -38,4 +46,3 @@ class DctpTvIE(InfoExtractor):
             'ext': 'flv'
         }
 
-

From b04fbd789c1efd5f918f81b5a5b9b6dafa806900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 22:49:42 +0600
Subject: [PATCH 643/974] [viddler] Modernize

---
 youtube_dl/extractor/viddler.py | 46 ++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index bacbad50a..9caee94e7 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -13,43 +13,43 @@ from ..compat import (
 class ViddlerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
     _TESTS = [{
-        "url": "http://www.viddler.com/v/43903784",
+        'url': 'http://www.viddler.com/v/43903784',
         'md5': 'ae43ad7cb59431ce043f0ff7fa13cbf4',
         'info_dict': {
             'id': '43903784',
             'ext': 'mp4',
-            "title": "Video Made Easy",
-            'description': 'You don\'t need to be a professional to make high-quality video content. Viddler provides some quick and easy tips on how to produce great video content with limited resources. ',
-            "uploader": "viddler",
+            'title': 'Video Made Easy',
+            'description': 'md5:6a697ebd844ff3093bd2e82c37b409cd',
+            'uploader': 'viddler',
             'timestamp': 1335371429,
             'upload_date': '20120425',
-            "duration": 100.89,
+            'duration': 100.89,
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
             'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
         }
     }, {
-        "url": "http://www.viddler.com/v/4d03aad9/",
-        "file": "4d03aad9.mp4",
-        "md5": "faa71fbf70c0bee7ab93076fd007f4b0",
-        "info_dict": {
+        'url': 'http://www.viddler.com/v/4d03aad9/',
+        'md5': 'faa71fbf70c0bee7ab93076fd007f4b0',
+        'info_dict': {
+            'id': '4d03aad9',
+            'ext': 'mp4',
+            'title': 'WALL-TO-GORTAT',
             'upload_date': '20150126',
             'uploader': 'deadspin',
-            'id': '4d03aad9',
             'timestamp': 1422285291,
-            'title': 'WALL-TO-GORTAT',
         }
     }, {
-        "url": "http://www.viddler.com/player/221ebbbd/0/",
-        "file": "221ebbbd.mp4",
-        "md5": "0defa2bd0ea613d14a6e9bd1db6be326",
-        "info_dict": {
+        'url': 'http://www.viddler.com/player/221ebbbd/0/',
+        'md5': '0defa2bd0ea613d14a6e9bd1db6be326',
+        'info_dict': {
+            'id': '221ebbbd',
+            'ext': 'mp4',
+            'title': 'LETeens-Grammar-snack-third-conditional',
+            'description': ' ',
             'upload_date': '20140929',
             'uploader': 'BCLETeens',
-            'id': '221ebbbd',
             'timestamp': 1411997190,
-            'title': 'LETeens-Grammar-snack-third-conditional',
-            'description': ' '
         }
     }]
 
@@ -68,7 +68,7 @@ class ViddlerIE(InfoExtractor):
             if filed.get('status', 'ready') != 'ready':
                 continue
             f = {
-                'format_id': filed['profile_id'],
+                'format_id': filed['profile_id'] or filed['profile_name'],
                 'format_note': filed['profile_name'],
                 'url': self._proto_relative_url(filed['url']),
                 'width': int_or_none(filed.get('width')),
@@ -82,15 +82,14 @@ class ViddlerIE(InfoExtractor):
             if filed.get('cdn_url'):
                 f = f.copy()
                 f['url'] = self._proto_relative_url(filed['cdn_url'], 'http:')
-                f['format_id'] = filed['profile_id'] + '-cdn'
+                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-cdn'
                 f['source_preference'] = 1
                 formats.append(f)
 
             if filed.get('html5_video_source'):
                 f = f.copy()
-                f['url'] = self._proto_relative_url(
-                    filed['html5_video_source'])
-                f['format_id'] = filed['profile_id'] + '-html5'
+                f['url'] = self._proto_relative_url(filed['html5_video_source'])
+                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-html5'
                 f['source_preference'] = 0
                 formats.append(f)
         self._sort_formats(formats)
@@ -99,7 +98,6 @@ class ViddlerIE(InfoExtractor):
             t.get('text') for t in data.get('tags', []) if 'text' in t]
 
         return {
-            '_type': 'video',
             'id': video_id,
             'title': data['title'],
             'formats': formats,

From e0d9f85aee92ebcb40ce688ed43172c718751e27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 22:56:06 +0600
Subject: [PATCH 644/974] Credit @HyShai for viddler fixes (#4794)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 1596a7548..20e620c35 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -106,3 +106,4 @@ Johan K. Jensen
 Yen Chi Hsuan
 Enam Mijbah Noor
 David Luhmer
+Shaya Goldberg

From cb454b333d91718a0c2b36c34c8b0d6858ff9505 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 23:07:37 +0600
Subject: [PATCH 645/974] [generic] Improve some regexes

---
 youtube_dl/extractor/generic.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a937a1e09..41884ed7a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -874,15 +874,15 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
 
         # Look for embedded Viddler player
-        mobj = (re.search(r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/embed/.+?)\1', webpage) or
-                re.search(r'<param[^>]+?value=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/player/.+?)\1', webpage))
-
+        mobj = re.search(
+            r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
+            webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
         # Look for Ooyala videos
-        mobj = (re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
-                re.search(r'OO.Player.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
+        mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
+                re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:
             return OoyalaIE._build_url_result(mobj.group('ec'))
 

From 18b4e9e79d6cc1ed21df7479273915ca3a763b80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 23:21:17 +0600
Subject: [PATCH 646/974] [viddler] Extract comment count

---
 youtube_dl/extractor/viddler.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 9caee94e7..ef104dc29 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -26,6 +26,7 @@ class ViddlerIE(InfoExtractor):
             'duration': 100.89,
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
+            'comment_count': int,
             'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
         }
     }, {
@@ -38,6 +39,8 @@ class ViddlerIE(InfoExtractor):
             'upload_date': '20150126',
             'uploader': 'deadspin',
             'timestamp': 1422285291,
+            'view_count': int,
+            'comment_count': int,
         }
     }, {
         'url': 'http://www.viddler.com/player/221ebbbd/0/',
@@ -50,6 +53,8 @@ class ViddlerIE(InfoExtractor):
             'upload_date': '20140929',
             'uploader': 'BCLETeens',
             'timestamp': 1411997190,
+            'view_count': int,
+            'comment_count': int,
         }
     }]
 
@@ -67,8 +72,9 @@ class ViddlerIE(InfoExtractor):
         for filed in data['files']:
             if filed.get('status', 'ready') != 'ready':
                 continue
+            format_id = filed.get('profile_id') or filed['profile_name']
             f = {
-                'format_id': filed['profile_id'] or filed['profile_name'],
+                'format_id': format_id,
                 'format_note': filed['profile_name'],
                 'url': self._proto_relative_url(filed['url']),
                 'width': int_or_none(filed.get('width')),
@@ -82,14 +88,14 @@ class ViddlerIE(InfoExtractor):
             if filed.get('cdn_url'):
                 f = f.copy()
                 f['url'] = self._proto_relative_url(filed['cdn_url'], 'http:')
-                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-cdn'
+                f['format_id'] = format_id + '-cdn'
                 f['source_preference'] = 1
                 formats.append(f)
 
             if filed.get('html5_video_source'):
                 f = f.copy()
                 f['url'] = self._proto_relative_url(filed['html5_video_source'])
-                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-html5'
+                f['format_id'] = format_id + '-html5'
                 f['source_preference'] = 0
                 formats.append(f)
         self._sort_formats(formats)
@@ -107,5 +113,6 @@ class ViddlerIE(InfoExtractor):
             'uploader': data.get('author'),
             'duration': float_or_none(data.get('length')),
             'view_count': int_or_none(data.get('view_count')),
+            'comment_count': int_or_none(data.get('comment_count')),
             'categories': categories,
         }

From 63be3b89894f26caf69d9667f8e7db5ecbcb03ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 23:58:14 +0600
Subject: [PATCH 647/974] [ivi] Modernize

---
 youtube_dl/extractor/ivi.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 7a400323d..e82594444 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -16,7 +16,7 @@ from ..utils import (
 class IviIE(InfoExtractor):
     IE_DESC = 'ivi.ru'
     IE_NAME = 'ivi'
-    _VALID_URL = r'https?://(?:www\.)?ivi\.ru/(?:watch/(?:[^/]+/)?|video/player\?.*?videoId=)(?P<videoid>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?ivi\.ru/(?:watch/(?:[^/]+/)?|video/player\?.*?videoId=)(?P<id>\d+)'
 
     _TESTS = [
         # Single movie
@@ -63,29 +63,34 @@ class IviIE(InfoExtractor):
         return int(m.group('commentcount')) if m is not None else 0
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
+        video_id = self._match_id(url)
 
         api_url = 'http://api.digitalaccess.ru/api/json/'
 
-        data = {'method': 'da.content.get',
-                'params': [video_id, {'site': 's183',
-                                      'referrer': 'http://www.ivi.ru/watch/%s' % video_id,
-                                      'contentid': video_id
-                                      }
-                           ]
+        data = {
+            'method': 'da.content.get',
+            'params': [
+                video_id, {
+                    'site': 's183',
+                    'referrer': 'http://www.ivi.ru/watch/%s' % video_id,
+                    'contentid': video_id
                 }
+            ]
+        }
 
         request = compat_urllib_request.Request(api_url, json.dumps(data))
 
-        video_json_page = self._download_webpage(request, video_id, 'Downloading video JSON')
+        video_json_page = self._download_webpage(
+            request, video_id, 'Downloading video JSON')
         video_json = json.loads(video_json_page)
 
         if 'error' in video_json:
             error = video_json['error']
             if error['origin'] == 'NoRedisValidData':
                 raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-            raise ExtractorError('Unable to download video %s: %s' % (video_id, error['message']), expected=True)
+            raise ExtractorError(
+                'Unable to download video %s: %s' % (video_id, error['message']),
+                expected=True)
 
         result = video_json['result']
 

From dcf53d440801505a27ee5615e3fb58b6a794bc73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jan 2015 00:44:40 +0600
Subject: [PATCH 648/974] [YoutubeDL] Set format_id for video+audio (Closes
 #3634)

---
 youtube_dl/YoutubeDL.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7f054cdff..4c238c555 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1074,6 +1074,7 @@ class YoutubeDL(object):
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
+                                'format_id': rf,
                                 'ext': formats_info[0]['ext'],
                                 'width': formats_info[0].get('width'),
                                 'height': formats_info[0].get('height'),

From 206dba27a4d134cf9a65b93f7a12bee033a80b18 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 29 Jan 2015 03:18:53 +0800
Subject: [PATCH 649/974] [NextMedia] Add new extractor

---
 youtube_dl/extractor/__init__.py  |   6 ++
 youtube_dl/extractor/nextmedia.py | 163 ++++++++++++++++++++++++++++++
 2 files changed, 169 insertions(+)
 create mode 100644 youtube_dl/extractor/nextmedia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..444af8b6e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -284,6 +284,12 @@ from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
+from .nextmedia import (
+    NextMediaIE,
+    NextMediaActionNewsIE,
+    AppleDailyRealtimeNewsIE,
+    AppleDailyAnimationNewsIE
+)
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhl import NHLIE, NHLVideocenterIE
diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
new file mode 100644
index 000000000..02dba4ef6
--- /dev/null
+++ b/youtube_dl/extractor/nextmedia.py
@@ -0,0 +1,163 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601
+
+
+class NextMediaIE(InfoExtractor):
+    _VALID_URL = r'http://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
+        'md5': 'dff9fad7009311c421176d1ac90bfe4f',
+        'info_dict': {
+            'id': '53109199',
+            'ext': 'mp4',
+            'title': '【佔領金鐘】50外國領事議員撐場 讚學生勇敢香港有希望',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:28222b9912b6665a21011b034c70fcc7',
+            'timestamp': 1415456273,
+            'upload_date': '20141108',
+        }
+    }]
+
+    _URL_PATTERN = r'\{ url: \'(.+)\' \}'
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        page = self._download_webpage(url, news_id)
+        return self._extract_from_nextmedia_page(news_id, url, page)
+
+    def _extract_from_nextmedia_page(self, news_id, url, page):
+        title = self._fetch_title(page)
+        video_url = self._search_regex(self._URL_PATTERN, page, 'video url')
+
+        attrs = {
+            'id': news_id,
+            'title': title,
+            'url': video_url,  # ext can be inferred from url
+            'thumbnail': self._fetch_thumbnail(page),
+            'description': self._fetch_description(page),
+        }
+
+        timestamp = self._fetch_timestamp(page)
+        if timestamp:
+            attrs['timestamp'] = timestamp
+        else:
+            attrs['upload_date'] = self._fetch_upload_date(url)
+
+        return attrs
+
+    def _fetch_title(self, page):
+        return self._og_search_title(page)
+
+    def _fetch_thumbnail(self, page):
+        return self._og_search_thumbnail(page)
+
+    def _fetch_timestamp(self, page):
+        dateCreated = self._search_regex('"dateCreated":"([^"]+)"', page, 'created time')
+        return parse_iso8601(dateCreated)
+
+    def _fetch_upload_date(self, url):
+        return self._search_regex(self._VALID_URL, url, 'upload date', group='date')
+
+    def _fetch_description(self, page):
+        return self._og_search_property('description', page)
+
+
+class NextMediaActionNewsIE(NextMediaIE):
+    _VALID_URL = r'http://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
+    _TESTS = [{
+        'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
+        'md5': '05fce8ffeed7a5e00665d4b7cf0f9201',
+        'info_dict': {
+            'id': '19009428',
+            'ext': 'mp4',
+            'title': '【壹週刊】細10年男友偷食　50歲邵美琪再失戀',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:cd802fad1f40fd9ea178c1e2af02d659',
+            'timestamp': 1421791200,
+            'upload_date': '20150120',
+        }
+    }]
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        actionnews_page = self._download_webpage(url, news_id)
+        article_url = self._og_search_url(actionnews_page)
+        article_page = self._download_webpage(article_url, news_id)
+        return self._extract_from_nextmedia_page(news_id, url, article_page)
+
+
+class AppleDailyRealtimeNewsIE(NextMediaIE):
+    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _TESTS = [{
+        'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
+        'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
+        'info_dict': {
+            'id': '36354694',
+            'ext': 'mp4',
+            'title': '周亭羽走過摩鐵陰霾2男陪吃 九把刀孤寒看醫生',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:b23787119933404ce515c6356a8c355c',
+            'upload_date': '20150128',
+        }
+    }, {
+        'url': 'http://www.appledaily.com.tw/realtimenews/article/strange/20150128/550549/%E4%B8%8D%E6%BB%BF%E8%A2%AB%E8%B8%A9%E8%85%B3%E3%80%80%E5%B1%B1%E6%9D%B1%E5%85%A9%E5%A4%A7%E5%AA%BD%E4%B8%80%E8%B7%AF%E6%89%93%E4%B8%8B%E8%BB%8A',
+        'md5': '86b4e9132d158279c7883822d94ccc49',
+        'info_dict': {
+            'id': '550549',
+            'ext': 'mp4',
+            'title': '不滿被踩腳　山東兩大媽一路打下車',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:2648aaf6fc4f401f6de35a91d111aa1d',
+            'upload_date': '20150128',
+        }
+    }]
+
+    _URL_PATTERN = r'\{url: \'(.+)\'\}'
+
+    def _fetch_title(self, page):
+        return self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title')
+
+    def _fetch_thumbnail(self, page):
+        return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)
+
+    def _fetch_timestamp(self, page):
+        return None
+
+
+class AppleDailyAnimationNewsIE(AppleDailyRealtimeNewsIE):
+    _VALID_URL = 'http://www.appledaily.com.tw/animation/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _TESTS = [{
+        'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003671',
+        'md5': '03df296d95dedc2d5886debbb80cb43f',
+        'info_dict': {
+            'id': '5003671',
+            'ext': 'mp4',
+            'title': '20正妹熱舞　《刀龍傳說Online》火辣上市',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:23c0aac567dc08c9c16a3161a2c2e3cd',
+            'upload_date': '20150128',
+        }
+    }, {
+        # No thumbnail
+        'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003673/',
+        'md5': 'b06182cd386ea7bc6115ec7ff0f72aeb',
+        'info_dict': {
+            'id': '5003673',
+            'ext': 'mp4',
+            'title': '半夜尿尿　好像會看到___',
+            'description': 'md5:61d2da7fe117fede148706cdb85ac066',
+            'upload_date': '20150128',
+        },
+        'expected_warnings': [
+            'video thumbnail',
+        ]
+    }]
+
+    def _fetch_title(self, page):
+        return self._html_search_meta('description', page, 'news title')
+
+    def _fetch_description(self, page):
+        return self._html_search_meta('description', page, 'news description')

From 367cc95aa76731aa951e949b325ac9c909af639d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 29 Jan 2015 03:49:56 +0800
Subject: [PATCH 650/974] [CtsNews] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ctsnews.py  | 51 ++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 youtube_dl/extractor/ctsnews.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..72c5e4973 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -82,6 +82,7 @@ from .crunchyroll import (
     CrunchyrollShowPlaylistIE
 )
 from .cspan import CSpanIE
+from .ctsnews import CtsNewsIE
 from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
new file mode 100644
index 000000000..e1d8c814e
--- /dev/null
+++ b/youtube_dl/extractor/ctsnews.py
@@ -0,0 +1,51 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601, ExtractorError
+
+
+class CtsNewsIE(InfoExtractor):
+    _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
+    _TESTS = [{
+        'url': 'http://news.cts.com.tw/cts/international/201309/201309031304098.html',
+        'md5': '3aee7e0df7cdff94e43581f54c22619e',
+        'info_dict': {
+            'id': '201309031304098',
+            'ext': 'mp4',
+            'title': '韓國31歲童顏男 貌如十多歲小孩',
+            'description': 'md5:f183feeba3752b683827aab71adad584',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1378205880,
+            'upload_date': '20130903',
+        }
+    }]
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        page = self._download_webpage(url, news_id)
+
+        if not self._search_regex(r'(CTSPlayer2)', page, 'CTSPlayer2 identifier', fatal=False):
+            raise ExtractorError('The news includes no videos!')
+
+        feed_pattern = r'(http://news.cts.com.tw/action/mp4feed.php\?news_id=\d+)'
+        feed_url = self._html_search_regex(feed_pattern, page, 'feed url')
+        feed_page = self._download_webpage(feed_url, news_id)
+
+        description = self._html_search_meta('description', page)
+        title = self._html_search_meta('title', page)
+        thumbnail = self._html_search_meta('image', page)
+
+        datetime_pattern = r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})'
+        datetime_str = self._html_search_regex(datetime_pattern, page, 'date and time')
+        time = (datetime_str + ':00+08:00').replace('/', '-')
+        timestamp = parse_iso8601(time, delimiter=' ')
+
+        return {
+            'id': news_id,
+            'title': title,
+            'description': description,
+            'url': feed_page,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp
+        }

From d20547610314e0821542fecd1872b5851bd8d5d7 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Thu, 29 Jan 2015 01:36:15 +0100
Subject: [PATCH 651/974] fix srmediathek description

---
 youtube_dl/extractor/srmediathek.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dl/extractor/srmediathek.py
index 666a7dcc8..5d583c720 100644
--- a/youtube_dl/extractor/srmediathek.py
+++ b/youtube_dl/extractor/srmediathek.py
@@ -8,7 +8,7 @@ from ..utils import js_to_json
 
 
 class SRMediathekIE(InfoExtractor):
-    IE_DESC = 'Süddeutscher Rundfunk'
+    IE_DESC = 'Saarländischer Rundfunk'
     _VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
 
     _TEST = {

From 4f264c02c73c8f7551bcdbc960c9635860003196 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 29 Jan 2015 13:37:17 +0100
Subject: [PATCH 652/974] [utils] YoutubeDLHTTPSHandler.https_open: pass all
 required arguments to do_open

With this change the '--no-check-certificate' works again (#4807).
---
 youtube_dl/utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b8c52af74..a4c9813ec 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -654,9 +654,14 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
         self._params = params
 
     def https_open(self, req):
+        kwargs = {}
+        if hasattr(self, '_context'):  # python > 2.6
+            kwargs['context'] = self._context
+        if hasattr(self, '_check_hostname'):  # python 3.x
+            kwargs['check_hostname'] = self._check_hostname
         return self.do_open(functools.partial(
             _create_http_connection, self, self._https_conn_class, True),
-            req)
+            req, **kwargs)
 
 
 def parse_iso8601(date_str, delimiter='T'):

From affd04a45dcc46f21b55e4d9419812c33b9aa9ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jan 2015 22:09:59 +0600
Subject: [PATCH 653/974] [xuite] Simplify and improve

---
 youtube_dl/extractor/xuite.py | 181 +++++++++++++++-------------------
 1 file changed, 82 insertions(+), 99 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 8466d4bc5..a9dbf8c2d 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -2,158 +2,141 @@
 from __future__ import unicode_literals
 
 import base64
+
 from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
-    compat_urllib_parse_unquote,
-    compat_parse_qs
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     parse_iso8601,
-    parse_duration
+    parse_duration,
 )
 
-# ref: http://stackoverflow.com/questions/475074/regex-to-parse-or-validate-base64-data
-REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
-
-CST_ZONE = +8  # China Standard Time
-
 
 class XuiteIE(InfoExtractor):
-    _VALID_URL = r'http://vlog.xuite.net/play/(?P<id>%s)(/.*)?' % REGEX_BASE64
+    _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
+    _VALID_URL = r'http://vlog.xuite.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
     _TESTS = [{
         # Audio
         'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',
         'md5': '63a42c705772aa53fd4c1a0027f86adf',
         'info_dict': {
-            'id': 'RGkzc1ZULTM4NjA5MTQuZmx2',
+            'id': '3860914',
             'ext': 'mp3',
-            'upload_date': '20110902',
-            'uploader_id': '15973816',
-            'uploader': '阿能',
-            'timestamp': 1314932940,
             'title': '孤單南半球-歐德陽',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 247.246,
+            'timestamp': 1314932940,
+            'upload_date': '20110902',
+            'uploader': '阿能',
+            'uploader_id': '15973816',
             'categories': ['個人短片'],
-            'duration': 247.246
-        }
-    }, {
-        # Audio with alternative form of url
-        'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
-        'md5': 'c91645f324de53d82ebb80930e1a73d2',
-        'info_dict': {
-            'id': 'S1dDUjdyLTMyOTc3NjcuZmx2',
-            'ext': 'mp3',
-            'upload_date': '20101226',
-            'uploader_id': '10102699',
-            'uploader': '蠍',
-            'timestamp': 1293367080,
-            'title': '孫燕姿-眼淚成詩',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'categories': ['個人短片'],
-            'duration': 223.19
-        }
+        },
     }, {
         # Video with only one format
         'url': 'http://vlog.xuite.net/play/TkRZNjhULTM0NDE2MjkuZmx2',
         'md5': 'c45737fc8ac5dc8ac2f92ecbcecf505e',
         'info_dict': {
-            'id': 'TkRZNjhULTM0NDE2MjkuZmx2',
+            'id': '3441629',
             'ext': 'mp4',
-            'upload_date': '20110306',
-            'uploader_id': '10400126',
-            'uploader': 'Valen',
-            'timestamp': 1299383640,
             'title': '孫燕姿 - 眼淚成詩',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 217.399,
+            'timestamp': 1299383640,
+            'upload_date': '20110306',
+            'uploader': 'Valen',
+            'uploader_id': '10400126',
             'categories': ['影視娛樂'],
-            'duration': 217.399
-        }
+        },
     }, {
         # Video with two formats
         'url': 'http://vlog.xuite.net/play/bWo1N1pLLTIxMzAxMTcwLmZsdg==',
         'md5': '1166e0f461efe55b62e26a2d2a68e6de',
         'info_dict': {
-            'id': 'bWo1N1pLLTIxMzAxMTcwLmZsdg==',
+            'id': '21301170',
             'ext': 'mp4',
-            'upload_date': '20150117',
-            'uploader_id': '242127761',
-            'uploader': '我只是想認真點',
-            'timestamp': 1421481240,
             'title': '暗殺教室 02',
+            'description': '字幕:【極影字幕社】',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1384.907,
+            'timestamp': 1421481240,
+            'upload_date': '20150117',
+            'uploader': '我只是想認真點',
+            'uploader_id': '242127761',
             'categories': ['電玩動漫'],
-            'duration': 1384.907
-        }
+        },
+    }, {
+        'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
+        'only_matching': True,
     }]
 
-    def _flv_config(self, media_id):
+    def _extract_flv_config(self, media_id):
         base64_media_id = base64.b64encode(media_id.encode('utf-8')).decode('utf-8')
-        flv_config_url = 'http://vlog.xuite.net/flash/player?media=' + base64_media_id
-        flv_config = self._download_xml(flv_config_url, 'flv config')
-
+        flv_config = self._download_xml(
+            'http://vlog.xuite.net/flash/player?media=%s' % base64_media_id,
+            'flv config')
         prop_dict = {}
         for prop in flv_config.findall('./property'):
             prop_id = base64.b64decode(prop.attrib['id']).decode('utf-8')
-
+            # CDATA may be empty in flv config
             if not prop.text:
-                continue  # CDATA may be empty in flv config
-
+                continue
             encoded_content = base64.b64decode(prop.text).decode('utf-8')
             prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content)
-
         return prop_dict
 
-    def _type_string(self, media_url):
-        query_string = compat_urlparse.urlparse(media_url).query
-        type_string = compat_parse_qs(query_string)['q'][0]
-        return type_string
-
-    def _guess_ext(self, media_url):
-        type_string = self._type_string(media_url)
-        if type_string == 'mp3':
-            return 'mp3'
-        elif type_string == '360' or type_string == '720':
-            return 'mp4'
-        else:
-            raise ExtractorError('Unknown type string %s' % type_string)
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        page = self._download_webpage(url, video_id)
-        media_id = self._html_search_regex(r'data-mediaid="(\d+)"', page, 'media id')
-        flv_config = self._flv_config(media_id)
+        webpage = self._download_webpage(url, video_id)
 
-        timestamp_local = parse_iso8601(flv_config['publish_datetime'], ' ')
-        timestamp_gmt = timestamp_local - CST_ZONE * 3600
+        error_msg = self._search_regex(
+            r'<div id="error-message-content">([^<]+)',
+            webpage, 'error message', default=None)
+        if error_msg:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_msg),
+                expected=True)
 
-        ret_attrs = {
-            'id': video_id,
-            'title': flv_config['title'],
-            'thumbnail': flv_config['thumb'],
-            'uploader': flv_config['author_name'],
-            'timestamp': timestamp_gmt,
-            'uploader_id': flv_config['author_id'],
-            'duration': parse_duration(flv_config['duration']),
-            'categories': [flv_config['category']]
+        video_id = self._html_search_regex(
+            r'data-mediaid="(\d+)"', webpage, 'media id')
+        flv_config = self._extract_flv_config(video_id)
+
+        FORMATS = {
+            'audio': 'mp3',
+            'video': 'mp4',
         }
 
-        if 'hq_src' in flv_config:
-            urls = [flv_config['src'], flv_config['hq_src']]
+        formats = []
+        for format_tag in ('src', 'hq_src'):
+            video_url = flv_config.get(format_tag)
+            if not video_url:
+                continue
+            format_id = self._search_regex(
+                r'\bq=(.+?)\b', video_url, 'format id', default=format_tag)
+            formats.append({
+                'url': video_url,
+                'ext': FORMATS.get(flv_config['type'], 'mp4'),
+                'format_id': format_id,
+                'height': int(format_id) if format_id.isnumeric() else None,
+            })
+        self._sort_formats(formats)
 
-            ret_attrs['formats'] = []
+        timestamp = flv_config.get('publish_datetime')
+        if timestamp:
+            timestamp = parse_iso8601(timestamp + ' +0800', ' ')
 
-            for url in urls:
-                ret_attrs['formats'].append({
-                    'url': url,
-                    'ext': self._guess_ext(url),
-                    'format_id': self._type_string(url),
-                    'height': int(self._type_string(url))
-                })
-        else:
-            ret_attrs['url'] = flv_config['src']
-            ret_attrs['ext'] = self._guess_ext(flv_config['src'])
+        category = flv_config.get('category')
+        categories = [category] if category else []
 
-        return ret_attrs
+        return {
+            'id': video_id,
+            'title': flv_config['title'],
+            'description': flv_config.get('description'),
+            'thumbnail': flv_config.get('thumb'),
+            'timestamp': timestamp,
+            'uploader': flv_config.get('author_name'),
+            'uploader_id': flv_config.get('author_id'),
+            'duration': parse_duration(flv_config.get('duration')),
+            'categories': categories,
+            'formats': formats,
+        }

From 219337990b8008eab81e1f9adf9eba20757f3cad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jan 2015 23:11:13 +0600
Subject: [PATCH 654/974] [xuite] Fix _VALID_URL

---
 youtube_dl/extractor/xuite.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index a9dbf8c2d..4971965f9 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class XuiteIE(InfoExtractor):
     _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
-    _VALID_URL = r'http://vlog.xuite.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
+    _VALID_URL = r'https?://vlog\.xuite\.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
     _TESTS = [{
         # Audio
         'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',

From a7a14d958604f5334413e2fc1872a8317d5e4884 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 29 Jan 2015 20:15:38 +0100
Subject: [PATCH 655/974] [YoutubeDL] set the 'thumbnails' field if the
 info_dict has the 'thumbnails' field

Since the '--write-thumbnail' uses the 'thumbnails' field and we didn't updated the info_dict, it wouldn't detect the thumbnail. (fixes #4812)
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4c238c555..14e92ddcf 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -958,7 +958,7 @@ class YoutubeDL(object):
         if thumbnails is None:
             thumbnail = info_dict.get('thumbnail')
             if thumbnail:
-                thumbnails = [{'url': thumbnail}]
+                info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
         if thumbnails:
             thumbnails.sort(key=lambda t: (
                 t.get('preference'), t.get('width'), t.get('height'),

From e683a48d0e8650e1d9e8c323cacb468b104ea7bc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 30 Jan 2015 05:38:05 +0800
Subject: [PATCH 656/974] [ctsnews] Detect youtube embedde videos

---
 youtube_dl/extractor/ctsnews.py | 75 +++++++++++++++++++++++++++------
 1 file changed, 63 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index e1d8c814e..35f3756f5 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -1,13 +1,27 @@
-# coding: utf-8
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
 from ..utils import parse_iso8601, ExtractorError
 
 
 class CtsNewsIE(InfoExtractor):
+    # https connection failed (Connection reset)
     _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
     _TESTS = [{
+        'url': 'http://news.cts.com.tw/cts/international/201501/201501291578109.html',
+        'md5': 'a9875cb790252b08431186d741beaabe',
+        'info_dict': {
+            'id': '201501291578109',
+            'ext': 'mp4',
+            'title': '以色列.真主黨交火 3人死亡',
+            'description': 'md5:95e9b295c898b7ff294f09d450178d7d',
+            'timestamp': 1422528540,
+            'upload_date': '20150129',
+        }
+    }, {
+        # News count not appear on page but still available in database
         'url': 'http://news.cts.com.tw/cts/international/201309/201309031304098.html',
         'md5': '3aee7e0df7cdff94e43581f54c22619e',
         'info_dict': {
@@ -19,33 +33,70 @@ class CtsNewsIE(InfoExtractor):
             'timestamp': 1378205880,
             'upload_date': '20130903',
         }
+    }, {
+        # With Youtube embedded video
+        'url': 'http://news.cts.com.tw/cts/money/201501/201501291578003.html',
+        'md5': '1d842c771dc94c8c3bca5af2cc1db9c5',
+        'add_ie': ['Youtube'],
+        'info_dict': {
+            'id': 'OVbfO7d0_hQ',
+            'ext': 'mp4',
+            'title': 'iPhone6熱銷 蘋果財報亮眼',
+            'description': 'md5:f395d4f485487bb0f992ed2c4b07aa7d',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150128',
+            'uploader_id': 'TBSCTS',
+            'uploader': '中華電視公司',
+        }
     }]
 
     def _real_extract(self, url):
         news_id = self._match_id(url)
         page = self._download_webpage(url, news_id)
 
-        if not self._search_regex(r'(CTSPlayer2)', page, 'CTSPlayer2 identifier', fatal=False):
-            raise ExtractorError('The news includes no videos!')
+        if self._search_regex(r'(CTSPlayer2)', page, 'CTSPlayer2 identifier', default=None):
+            feed_url = self._html_search_regex(
+                r'(http://news\.cts\.com\.tw/action/mp4feed\.php\?news_id=\d+)',
+                page, 'feed url')
+            video_url = self._download_webpage(feed_url, news_id)
+        else:
+            self.to_screen('Not CTSPlayer video, trying Youtube...')
+            youtube_url = self._search_regex(
+                r'src="(//www\.youtube\.com/embed/[^"]+)"', page, 'youtube url',
+                default=None)
+            if not youtube_url:
+                raise ExtractorError('The news includes no videos!', expected=True)
 
-        feed_pattern = r'(http://news.cts.com.tw/action/mp4feed.php\?news_id=\d+)'
-        feed_url = self._html_search_regex(feed_pattern, page, 'feed url')
-        feed_page = self._download_webpage(feed_url, news_id)
+            return {
+                '_type': 'url',
+                'url': youtube_url,
+                'ie_key': 'Youtube',
+            }
 
         description = self._html_search_meta('description', page)
         title = self._html_search_meta('title', page)
         thumbnail = self._html_search_meta('image', page)
 
-        datetime_pattern = r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})'
-        datetime_str = self._html_search_regex(datetime_pattern, page, 'date and time')
-        time = (datetime_str + ':00+08:00').replace('/', '-')
-        timestamp = parse_iso8601(time, delimiter=' ')
+        datetime_str = self._html_search_regex(
+            r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})', page, 'date and time')
+        # Transform into ISO 8601 format with timezone info
+        datetime_str = datetime_str.replace('/', '-') + ':00+0800'
+        timestamp = parse_iso8601(datetime_str, delimiter=' ')
+
+        # Note: the news count may decrease as time goes by
+        # It should be a bug in CTS website
+        req = compat_urllib_request.Request(
+            'http://news.cts.com.tw/action/news_count.php?callback=cb&news_id=' + news_id)
+        req.add_header('Referer', url)
+        newscount_page = self._download_webpage(req, news_id)
+        news_count = self._search_regex(r'cb\((\d+)\)', newscount_page, 'news count')
 
         return {
             'id': news_id,
+            'url': video_url,
             'title': title,
             'description': description,
-            'url': feed_page,
             'thumbnail': thumbnail,
-            'timestamp': timestamp
+            'timestamp': timestamp,
+            'view_count': news_count,
         }

From f345fe9db77a900a603d753c73c5566d62c1cff9 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Thu, 29 Jan 2015 23:32:23 +0100
Subject: [PATCH 657/974] [dctp] fix python 2.6 compatibility

---
 youtube_dl/extractor/dctp.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 9b687ef43..31bcd35c3 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -20,10 +20,10 @@ class DctpTvIE(InfoExtractor):
         version_json = self._download_json(base_url + 'version.json', video_id)
         version = version_json['version_name']
         info_json = self._download_json(
-            '{}{}/restapi/slugs/{}.json'.format(base_url, version, video_id), video_id)
+            '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id), video_id)
         object_id = info_json['object_id']
         meta_json = self._download_json(
-            '{}{}/restapi/media/{}.json'.format(base_url, version, object_id), video_id)
+            '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id), video_id)
         uuid = meta_json['uuid']
         title = meta_json['title']
         wide = meta_json['is_wide']
@@ -31,7 +31,7 @@ class DctpTvIE(InfoExtractor):
             ratio = '16x9'
         else:
             ratio = '4x3'
-        play_path = 'mp4:{}_dctp_0500_{}.m4v'.format(uuid, ratio)
+        play_path = 'mp4:{0}_dctp_0500_{1}.m4v'.format(uuid, ratio)
 
         servers_json = self._download_json('http://www.dctp.tv/streaming_servers/', video_id)
         url = servers_json[0]['endpoint']

From 87673cd4389f6798bb438bb46a306dc08c906cf5 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Thu, 29 Jan 2015 23:34:56 +0100
Subject: [PATCH 658/974] [dctp] follow id conventions

---
 youtube_dl/extractor/dctp.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 31bcd35c3..752cff978 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -37,12 +37,13 @@ class DctpTvIE(InfoExtractor):
         url = servers_json[0]['endpoint']
 
         return {
-            'id': video_id,
+            'id': object_id,
             'title': title,
             'format': 'rtmp',
             'url': url,
             'play_path': play_path,
             'real_time': True,
-            'ext': 'flv'
+            'ext': 'flv',
+            'display_id': video_id
         }
 

From 75a4fc5b7233d4cb4376b205c585eb4d3d486241 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Fri, 30 Jan 2015 00:35:53 +0100
Subject: [PATCH 659/974] [dctp] fix test

---
 youtube_dl/extractor/dctp.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 752cff978..c80d28865 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -9,7 +9,8 @@ class DctpTvIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
         'info_dict': {
-            'id': 'videoinstallation-fuer-eine-kaufhausfassade',
+            'id': '1324',
+            'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
             'ext': 'flv',
             'title': 'Videoinstallation für eine Kaufhausfassade'}
         }
@@ -21,7 +22,7 @@ class DctpTvIE(InfoExtractor):
         version = version_json['version_name']
         info_json = self._download_json(
             '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id), video_id)
-        object_id = info_json['object_id']
+        object_id = str(info_json['object_id'])
         meta_json = self._download_json(
             '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id), video_id)
         uuid = meta_json['uuid']

From 7bb3ceb4c731c9783d2c565204cbc1071c3545ed Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Fri, 30 Jan 2015 00:47:43 +0100
Subject: [PATCH 660/974] [dctp] prefix real_time parameter with rtmp_

---
 youtube_dl/downloader/rtmp.py | 2 +-
 youtube_dl/extractor/dctp.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6cb1bfc50..f7eeb6f43 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -104,9 +104,9 @@ class RtmpFD(FileDownloader):
         live = info_dict.get('rtmp_live', False)
         conn = info_dict.get('rtmp_conn', None)
         protocol = info_dict.get('rtmp_protocol', None)
+        real_time = info_dict.get('rtmp_real_time', False)
         no_resume = info_dict.get('no_resume', False)
         continue_dl = info_dict.get('continuedl', False)
-        real_time = info_dict.get('real_time', False)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index c80d28865..8a77f2b66 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -43,7 +43,7 @@ class DctpTvIE(InfoExtractor):
             'format': 'rtmp',
             'url': url,
             'play_path': play_path,
-            'real_time': True,
+            'rtmp_real_time': True,
             'ext': 'flv',
             'display_id': video_id
         }

From adc0ae3ceb9371eb23a9a4d81d0e134ac861723e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 01:56:15 +0100
Subject: [PATCH 661/974] [__init__] Provide a better error messages if URLs
 are missing (Closes #4813)

---
 youtube_dl/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 71d2c6f35..e90679ff9 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -361,7 +361,9 @@ def _real_main(argv=None):
                 sys.exit()
 
             ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
-            parser.error('you must provide at least one URL')
+            parser.error(
+                'You must provide at least one URL.\n'
+                'Type youtube-dl --help to see a list of all options.')
 
         try:
             if opts.load_info_filename is not None:

From 4fe8495a23271e164a1ce618682d50f927adc075 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 02:13:37 +0100
Subject: [PATCH 662/974] [viddler] PEP8

---
 youtube_dl/extractor/viddler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index ef104dc29..8516a2940 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -6,7 +6,7 @@ from ..utils import (
     int_or_none,
 )
 from ..compat import (
-   compat_urllib_request
+    compat_urllib_request
 )
 
 

From 83fda3c000a680317cfc9bb6fec899beb8bca773 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 02:57:37 +0100
Subject: [PATCH 663/974] Add a test for --no-check-certificate

---
 test/test_http.py    | 71 ++++++++++++++++++++++++++++++++++++++++++++
 test/testcert.pem    | 52 ++++++++++++++++++++++++++++++++
 youtube_dl/compat.py |  6 ++++
 3 files changed, 129 insertions(+)
 create mode 100644 test/test_http.py
 create mode 100644 test/testcert.pem

diff --git a/test/test_http.py b/test/test_http.py
new file mode 100644
index 000000000..5cce5b3ae
--- /dev/null
+++ b/test/test_http.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl import YoutubeDL
+from youtube_dl.compat import compat_http_server
+import ssl
+import threading
+
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+
+class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
+    def log_message(self, format, *args):
+        pass
+
+    def do_GET(self):
+        if self.path == '/video.html':
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.end_headers()
+            self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
+        elif self.path == '/vid.mp4':
+            self.send_response(200)
+            self.send_header('Content-Type', 'video/mp4')
+            self.end_headers()
+            self.wfile.write(b'\x00\x00\x00\x00\x20\x66\x74[video]')
+        else:
+            assert False
+
+
+class FakeLogger(object):
+    def debug(self, msg):
+        pass
+
+    def warning(self, msg):
+        pass
+
+    def error(self, msg):
+        pass
+
+
+class TestHTTP(unittest.TestCase):
+    def setUp(self):
+        certfn = os.path.join(TEST_DIR, 'testcert.pem')
+        self.httpd = compat_http_server.HTTPServer(
+            ('localhost', 0), HTTPTestRequestHandler)
+        self.httpd.socket = ssl.wrap_socket(
+            self.httpd.socket, certfile=certfn, server_side=True)
+        self.port = self.httpd.socket.getsockname()[1]
+        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
+        self.server_thread.daemon = True
+        self.server_thread.start()
+
+    def test_nocheckcertificate(self):
+        if sys.version_info >= (2, 7, 9):  # No certificate checking anyways
+            ydl = YoutubeDL({'logger': FakeLogger()})
+            self.assertRaises(
+                Exception,
+                ydl.extract_info, 'https://localhost:%d/video.html' % self.port)
+
+        ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True})
+        r = ydl.extract_info('https://localhost:%d/video.html' % self.port)
+        self.assertEqual(r['url'], 'https://localhost:%d/vid.mp4' % self.port)
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/testcert.pem b/test/testcert.pem
new file mode 100644
index 000000000..b3e0f00c7
--- /dev/null
+++ b/test/testcert.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMF0bAzaHAdIyB
+HRmnIp4vv40lGqEePmWqicCl0QZ0wsb5dNysSxSa7330M2QeQopGfdaUYF1uTcNp
+Qx6ECgBSfg+RrOBI7r/u4F+sKX8MUXVaf/5QoBUrGNGSn/pp7HMGOuQqO6BVg4+h
+A1ySSwUG8mZItLRry1ISyErmW8b9xlqfd97uLME/5tX+sMelRFjUbAx8A4CK58Ev
+mMguHVTlXzx5RMdYcf1VScYcjlV/qA45uzP8zwI5aigfcmUD+tbGuQRhKxUhmw0J
+aobtOR6+JSOAULW5gYa/egE4dWLwbyM6b6eFbdnjlQzEA1EW7ChMPAW/Mo83KyiP
+tKMCSQulAgMBAAECggEALCfBDAexPjU5DNoh6bIorUXxIJzxTNzNHCdvgbCGiA54
+BBKPh8s6qwazpnjT6WQWDIg/O5zZufqjE4wM9x4+0Zoqfib742ucJO9wY4way6x4
+Clt0xzbLPabB+MoZ4H7ip+9n2+dImhe7pGdYyOHoNYeOL57BBi1YFW42Hj6u/8pd
+63YCXisto3Rz1YvRQVjwsrS+cRKZlzAFQRviL30jav7Wh1aWEfcXxjj4zhm8pJdk
+ITGtq6howz57M0NtX6hZnfe8ywzTnDFIGKIMA2cYHuYJcBh9bc4tCGubTvTKK9UE
+8fM+f6UbfGqfpKCq1mcgs0XMoFDSzKS9+mSJn0+5JQKBgQD+OCKaeH3Yzw5zGnlw
+XuQfMJGNcgNr+ImjmvzUAC2fAZUJLAcQueE5kzMv5Fmd+EFE2CEX1Vit3tg0SXvA
+G+bq609doILHMA03JHnV1npO/YNIhG3AAtJlKYGxQNfWH9mflYj9mEui8ZFxG52o
+zWhHYuifOjjZszUR+/eio6NPzwKBgQDNhUBTrT8LIX4SE/EFUiTlYmWIvOMgXYvN
+8Cm3IRNQ/yyphZaXEU0eJzfX5uCDfSVOgd6YM/2pRah+t+1Hvey4H8e0GVTu5wMP
+gkkqwKPGIR1YOmlw6ippqwvoJD7LuYrm6Q4D6e1PvkjwCq6lEndrOPmPrrXNd0JJ
+XO60y3U2SwKBgQDLkyZarryQXxcCI6Q10Tc6pskYDMIit095PUbTeiUOXNT9GE28
+Hi32ziLCakk9kCysNasii81MxtQ54tJ/f5iGbNMMddnkKl2a19Hc5LjjAm4cJzg/
+98KGEhvyVqvAo5bBDZ06/rcrD+lZOzUglQS5jcIcqCIYa0LHWQ/wJLxFzwKBgFcZ
+1SRhdSmDfUmuF+S4ZpistflYjC3IV5rk4NkS9HvMWaJS0nqdw4A3AMzItXgkjq4S
+DkOVLTkTI5Do5HAWRv/VwC5M2hkR4NMu1VGAKSisGiKtRsirBWSZMEenLNHshbjN
+Jrpz5rZ4H7NT46ZkCCZyFBpX4gb9NyOedjA7Via3AoGARF8RxbYjnEGGFuhnbrJB
+FTPR0vaL4faY3lOgRZ8jOG9V2c9Hzi/y8a8TU4C11jnJSDqYCXBTd5XN28npYxtD
+pjRsCwy6ze+yvYXPO7C978eMG3YRyj366NXUxnXN59ibwe/lxi2OD9z8J1LEdF6z
+VJua1Wn8HKxnXMI61DhTCSo=
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIEEzCCAvugAwIBAgIJAK1haYi6gmSKMA0GCSqGSIb3DQEBCwUAMIGeMQswCQYD
+VQQGEwJERTEMMAoGA1UECAwDTlJXMRQwEgYDVQQHDAtEdWVzc2VsZG9yZjEbMBkG
+A1UECgwSeW91dHViZS1kbCBwcm9qZWN0MRkwFwYDVQQLDBB5b3V0dWJlLWRsIHRl
+c3RzMRIwEAYDVQQDDAlsb2NhbGhvc3QxHzAdBgkqhkiG9w0BCQEWEHBoaWhhZ0Bw
+aGloYWcuZGUwIBcNMTUwMTMwMDExNTA4WhgPMjExNTAxMDYwMTE1MDhaMIGeMQsw
+CQYDVQQGEwJERTEMMAoGA1UECAwDTlJXMRQwEgYDVQQHDAtEdWVzc2VsZG9yZjEb
+MBkGA1UECgwSeW91dHViZS1kbCBwcm9qZWN0MRkwFwYDVQQLDBB5b3V0dWJlLWRs
+IHRlc3RzMRIwEAYDVQQDDAlsb2NhbGhvc3QxHzAdBgkqhkiG9w0BCQEWEHBoaWhh
+Z0BwaGloYWcuZGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDMF0bA
+zaHAdIyBHRmnIp4vv40lGqEePmWqicCl0QZ0wsb5dNysSxSa7330M2QeQopGfdaU
+YF1uTcNpQx6ECgBSfg+RrOBI7r/u4F+sKX8MUXVaf/5QoBUrGNGSn/pp7HMGOuQq
+O6BVg4+hA1ySSwUG8mZItLRry1ISyErmW8b9xlqfd97uLME/5tX+sMelRFjUbAx8
+A4CK58EvmMguHVTlXzx5RMdYcf1VScYcjlV/qA45uzP8zwI5aigfcmUD+tbGuQRh
+KxUhmw0JaobtOR6+JSOAULW5gYa/egE4dWLwbyM6b6eFbdnjlQzEA1EW7ChMPAW/
+Mo83KyiPtKMCSQulAgMBAAGjUDBOMB0GA1UdDgQWBBTBUZoqhQkzHQ6xNgZfFxOd
+ZEVt8TAfBgNVHSMEGDAWgBTBUZoqhQkzHQ6xNgZfFxOdZEVt8TAMBgNVHRMEBTAD
+AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCUOCl3T/J9B08Z+ijfOJAtkbUaEHuVZb4x
+5EpZSy2ZbkLvtsftMFieHVNXn9dDswQc5qjYStCC4o60LKw4M6Y63FRsAZ/DNaqb
+PY3jyCyuugZ8/sNf50vHYkAcF7SQYqOQFQX4TQsNUk2xMJIt7H0ErQFmkf/u3dg6
+cy89zkT462IwxzSG7NNhIlRkL9o5qg+Y1mF9eZA1B0rcL6hO24PPTHOd90HDChBu
+SZ6XMi/LzYQSTf0Vg2R+uMIVlzSlkdcZ6sqVnnqeLL8dFyIa4e9sj/D4ZCYP8Mqe
+Z73H5/NNhmwCHRqVUTgm307xblQaWGhwAiDkaRvRW2aJQ0qGEdZK
+-----END CERTIFICATE-----
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 4453b34fc..497ca52de 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -71,6 +71,11 @@ try:
 except ImportError:
     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
 
+try:
+    import http.server as compat_http_server
+except ImportError:
+    import BaseHTTPServer as compat_http_server
+
 try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
 except ImportError:
@@ -365,6 +370,7 @@ __all__ = [
     'compat_html_entities',
     'compat_html_parser',
     'compat_http_client',
+    'compat_http_server',
     'compat_kwargs',
     'compat_ord',
     'compat_parse_qs',

From 03d8d4df385bbf1e4cf1f63609d06dc4ad11313a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:06:40 +0100
Subject: [PATCH 664/974] [test_http] PEP8

---
 test/test_http.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_http.py b/test/test_http.py
index 5cce5b3ae..bd4d46fef 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -14,6 +14,7 @@ import threading
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
+
 class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
     def log_message(self, format, *args):
         pass

From ec4161a57d66c92ea7460ac44dda13aadfb85f21 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:06:51 +0100
Subject: [PATCH 665/974] [ctsnews] Remove news count check (#4802)

That's the number of articles (or at least some constant value); view count is the number of views of the video!
Also add a better description so that we can distinguish the two downloads.
---
 youtube_dl/extractor/ctsnews.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index 35f3756f5..f769dc031 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -58,7 +58,8 @@ class CtsNewsIE(InfoExtractor):
             feed_url = self._html_search_regex(
                 r'(http://news\.cts\.com\.tw/action/mp4feed\.php\?news_id=\d+)',
                 page, 'feed url')
-            video_url = self._download_webpage(feed_url, news_id)
+            video_url = self._download_webpage(
+                feed_url, news_id, note='Fetching feed')
         else:
             self.to_screen('Not CTSPlayer video, trying Youtube...')
             youtube_url = self._search_regex(
@@ -83,14 +84,6 @@ class CtsNewsIE(InfoExtractor):
         datetime_str = datetime_str.replace('/', '-') + ':00+0800'
         timestamp = parse_iso8601(datetime_str, delimiter=' ')
 
-        # Note: the news count may decrease as time goes by
-        # It should be a bug in CTS website
-        req = compat_urllib_request.Request(
-            'http://news.cts.com.tw/action/news_count.php?callback=cb&news_id=' + news_id)
-        req.add_header('Referer', url)
-        newscount_page = self._download_webpage(req, news_id)
-        news_count = self._search_regex(r'cb\((\d+)\)', newscount_page, 'news count')
-
         return {
             'id': news_id,
             'url': video_url,
@@ -98,5 +91,4 @@ class CtsNewsIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'timestamp': timestamp,
-            'view_count': news_count,
         }

From 3198291f26e215a7b26906a630d42a30615869b4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:09:51 +0100
Subject: [PATCH 666/974] [ctsnews] Remove unused import

---
 youtube_dl/extractor/ctsnews.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index f769dc031..0226f8036 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import parse_iso8601, ExtractorError
 
 

From f452f72c6ba0cc3fe560e7449fcb95e804da5b69 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:12:12 +0100
Subject: [PATCH 667/974] Credit @yan12125 for ctsnews and nextmedia

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 20e620c35..2203c4b63 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -107,3 +107,4 @@ Yen Chi Hsuan
 Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
+Yen Chi Hsuan

From 6ebb0dca9f40ecf9fb82882c097f42f497c70cda Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:15:34 +0100
Subject: [PATCH 668/974] [dctp] Improve output (#4796)

---
 youtube_dl/extractor/dctp.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 8a77f2b66..dbedafa7d 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -2,10 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 
 
 class DctpTvIE(InfoExtractor):
-    _VALID_URL = r'^http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+    _VALID_URL = r'http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
         'info_dict': {
@@ -18,13 +19,17 @@ class DctpTvIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         base_url = 'http://dctp-ivms2-restapi.s3.amazonaws.com/'
-        version_json = self._download_json(base_url + 'version.json', video_id)
+        version_json = self._download_json(
+            base_url + 'version.json',
+            video_id, note='Determining file version')
         version = version_json['version_name']
         info_json = self._download_json(
-            '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id), video_id)
-        object_id = str(info_json['object_id'])
+            '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id),
+            video_id, note='Fetching object ID')
+        object_id = compat_str(info_json['object_id'])
         meta_json = self._download_json(
-            '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id), video_id)
+            '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id),
+            video_id, note='Downloading metadata')
         uuid = meta_json['uuid']
         title = meta_json['title']
         wide = meta_json['is_wide']
@@ -34,7 +39,9 @@ class DctpTvIE(InfoExtractor):
             ratio = '4x3'
         play_path = 'mp4:{0}_dctp_0500_{1}.m4v'.format(uuid, ratio)
 
-        servers_json = self._download_json('http://www.dctp.tv/streaming_servers/', video_id)
+        servers_json = self._download_json(
+            'http://www.dctp.tv/streaming_servers/',
+            video_id, note='Downloading server list')
         url = servers_json[0]['endpoint']
 
         return {

From 1798791df11920928818161c48fc2b12334e1e52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:16:36 +0100
Subject: [PATCH 669/974] Credit @bastik for dctp.tv (#4796)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 2203c4b63..4a70e7cca 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -108,3 +108,4 @@ Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
 Yen Chi Hsuan
+Paul Hartmann

From 83fddfd493ffcbb7256f0f1a467cc48ca2191090 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:45:06 +0100
Subject: [PATCH 670/974] [dctp] PEP8

---
 youtube_dl/extractor/dctp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index dbedafa7d..6ed354379 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -13,8 +13,9 @@ class DctpTvIE(InfoExtractor):
             'id': '1324',
             'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
             'ext': 'flv',
-            'title': 'Videoinstallation für eine Kaufhausfassade'}
+            'title': 'Videoinstallation für eine Kaufhausfassade'
         }
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -54,4 +55,3 @@ class DctpTvIE(InfoExtractor):
             'ext': 'flv',
             'display_id': video_id
         }
-

From c1708b89c09d7288213110fba859f4c5e0339c31 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:45:29 +0100
Subject: [PATCH 671/974] [youtube:truncated_url] Add hl= forms (#4799)

---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b7b91f354..bc5fa0b39 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1688,6 +1688,7 @@ class YoutubeTruncatedURLIE(InfoExtractor):
             feature=[a-z_]+|
             annotation_id=annotation_[^&]+|
             x-yt-cl=[0-9]+|
+            hl=[^&]*|
         )?
         |
             attribution_link\?a=[^&]+
@@ -1707,6 +1708,9 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     }, {
         'url': 'https://www.youtube.com/watch?feature=foo',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?hl=en-GB',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 5a8a29cfea8da1afaa94bd2d5be5c1bf72b2d17c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:46:57 +0100
Subject: [PATCH 672/974] release 2015.01.30

---
 README.md             | 6 +++---
 youtube_dl/version.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 0580ab2a3..314f9a8b2 100644
--- a/README.md
+++ b/README.md
@@ -292,9 +292,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      video results by putting a condition in
                                      brackets, as in -f "best[height=720]" (or
                                      -f "[filesize>10M]").  This works for
-                                     filesize, height, width, tbr, abr, and vbr
-                                     and the comparisons <, <=, >, >=, =, != .
-                                     Formats for which the value is not known
+                                     filesize, height, width, tbr, abr, vbr, and
+                                     fps and the comparisons <, <=, >, >=, =, !=
+                                     . Formats for which the value is not known
                                      are excluded unless you put a question mark
                                      (?) after the operator. You can combine
                                      format filters, so  -f "[height <=?
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0d59cddd7..4c75db08f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.25'
+__version__ = '2015.01.30'

From 5d6677ca28147495ffbeaf5bd49647e611cf50fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:50:40 +0100
Subject: [PATCH 673/974] release 2015.01.30.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4c75db08f..a5d558520 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.30'
+__version__ = '2015.01.30.1'

From e2a08185c6b9b3f1f4b7e78138b2b6735d6b0184 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 04:17:44 +0100
Subject: [PATCH 674/974] [README] Add an FAQ for YouTube IDs starting with a
 dash (Closes #4800)

---
 README.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/README.md b/README.md
index 314f9a8b2..09da0129e 100644
--- a/README.md
+++ b/README.md
@@ -525,6 +525,13 @@ From then on, after restarting your shell, you will be able to access both youtu
 
 Use the `-o` to specify an [output template](#output-template), for example `-o "/home/user/videos/%(title)s-%(id)s.%(ext)s"`. If you want this for all of your downloads, put the option into your [configuration file](#configuration).
 
+### How do I download a video starting with a `-` ?
+
+Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the options with `--`:
+
+    youtube-dl -- -wNyEUrxzFU
+    youtube-dl "http://www.youtube.com/watch?v=-wNyEUrxzFU"
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.

From 6449cd807e07e04eb88b1a34063e00677de0713e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 04:43:50 +0100
Subject: [PATCH 675/974] [youtube] Fall back to embed webpage when content is
 blocked on main (Fixes #4717)

---
 youtube_dl/extractor/youtube.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bc5fa0b39..a9e444351 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -28,6 +28,7 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
+    js_to_json,
     OnDemandPagedList,
     orderedSet,
     unescapeHTML,
@@ -809,6 +810,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             player_url = None
 
         # Get video info
+        embed_webpage = None
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
@@ -1016,10 +1018,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     url += '&signature=' + url_data['sig'][0]
                 elif 's' in url_data:
                     encrypted_sig = url_data['s'][0]
+                    ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
 
                     jsplayer_url_json = self._search_regex(
-                        r'"assets":.+?"js":\s*("[^"]+")',
-                        embed_webpage if age_gate else video_webpage, 'JS player URL')
+                        ASSETS_RE,
+                        embed_webpage if age_gate else video_webpage,
+                        'JS player URL (1)', default=None)
+                    if not jsplayer_url_json and not age_gate:
+                        # We need the embed website after all
+                        if embed_webpage is None:
+                            embed_url = proto + '://www.youtube.com/embed/%s' % video_id
+                            embed_webpage = self._download_webpage(
+                                embed_url, video_id, 'Downloading embed webpage')
+                        jsplayer_url_json = self._search_regex(
+                            ASSETS_RE, embed_webpage, 'JS player URL')
+
                     player_url = json.loads(jsplayer_url_json)
                     if player_url is None:
                         player_url_json = self._search_regex(

From c03844a4ec0bffef4803394545c7a5cc16dc70e1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 04:44:00 +0100
Subject: [PATCH 676/974] release 2015.01.30.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a5d558520..5e6288e8e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.30.1'
+__version__ = '2015.01.30.2'

From 52585fd6dcd5020f2800d133588d7232419dddc0 Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 18:41:40 +1100
Subject: [PATCH 677/974] The opening curly brace `{` is a regex reserved
 [control character](http://stackoverflow.com/a/400316/1106367), so it needs
 to be escaped.

---
 youtube_dl/extractor/generic.py     | 2 +-
 youtube_dl/extractor/grooveshark.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 41884ed7a..a6f102a4e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1073,7 +1073,7 @@ class GenericIE(InfoExtractor):
             found = filter_video(re.findall(r'''(?xs)
                 flowplayer\("[^"]+",\s*
                     \{[^}]+?\}\s*,
-                    \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
+                    \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
                         ["']?url["']?\s*:\s*["']([^"']+)["']
             ''', webpage))
         if not found:
diff --git a/youtube_dl/extractor/grooveshark.py b/youtube_dl/extractor/grooveshark.py
index fff74a70a..848d17beb 100644
--- a/youtube_dl/extractor/grooveshark.py
+++ b/youtube_dl/extractor/grooveshark.py
@@ -83,7 +83,7 @@ class GroovesharkIE(InfoExtractor):
         return compat_urlparse.urlunparse((uri.scheme, uri.netloc, obj['attrs']['data'], None, None, None))
 
     def _transform_bootstrap(self, js):
-        return re.split('(?m)^\s*try\s*{', js)[0] \
+        return re.split('(?m)^\s*try\s*\{', js)[0] \
                  .split(' = ', 1)[1].strip().rstrip(';')
 
     def _transform_meta(self, js):

From 1d2daaea6372f5119be8fc21637a275fe93a52ff Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 20:10:12 +1100
Subject: [PATCH 678/974] Simplify list creation

---
 youtube_dl/extractor/cinchcast.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dl/extractor/cinchcast.py
index 0c9a24bef..e23c515d9 100644
--- a/youtube_dl/extractor/cinchcast.py
+++ b/youtube_dl/extractor/cinchcast.py
@@ -28,12 +28,7 @@ class CinchcastIE(InfoExtractor):
             item, './{http://developer.longtailvideo.com/trac/}date')
         upload_date = unified_strdate(date_str, day_first=False)
         # duration is present but wrong
-        formats = []
-        formats.append({
-            'format_id': 'main',
-            'url': item.find(
-                './{http://search.yahoo.com/mrss/}content').attrib['url'],
-        })
+        formats = [{'format_id': 'main', 'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url']}]
         backup_url = xpath_text(
             item, './{http://developer.longtailvideo.com/trac/}backupContent')
         if backup_url:

From 5468ff4d917eeb292ce0bfe7cf13eef75cccae19 Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 20:11:51 +1100
Subject: [PATCH 679/974] Remove duplicate dictionary keys

---
 youtube_dl/YoutubeDL.py               | 1 -
 youtube_dl/extractor/appletrailers.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 14e92ddcf..ce3abaee3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1075,7 +1075,6 @@ class YoutubeDL(object):
                                 'requested_formats': formats_info,
                                 'format': rf,
                                 'format_id': rf,
-                                'ext': formats_info[0]['ext'],
                                 'width': formats_info[0].get('width'),
                                 'height': formats_info[0].get('height'),
                                 'resolution': formats_info[0].get('resolution'),
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 70621946d..287f71e07 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -122,7 +122,6 @@ class AppleTrailersIE(InfoExtractor):
             playlist.append({
                 '_type': 'video',
                 'id': video_id,
-                'title': title,
                 'formats': formats,
                 'title': title,
                 'duration': duration,

From 749f2ca044dc32c6a3ae9e9c55e4a22b091e2635 Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 20:35:20 +1100
Subject: [PATCH 680/974] Smotri info extractor: removed unreachable code and
 updated old md5 for test video

---
 youtube_dl/extractor/smotri.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index e94f41362..24746a09a 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -108,7 +108,7 @@ class SmotriIE(InfoExtractor):
         # swf player
         {
             'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',
-            'md5': '4d47034979d9390d14acdf59c4935bc2',
+            'md5': '31099eeb4bc906712c5f40092045108d',
             'info_dict': {
                 'id': 'v9188090500',
                 'ext': 'mp4',
@@ -139,9 +139,6 @@ class SmotriIE(InfoExtractor):
     def _search_meta(self, name, html, display_name=None):
         if display_name is None:
             display_name = name
-        return self._html_search_regex(
-            r'<meta itemprop="%s" content="([^"]+)" />' % re.escape(name),
-            html, display_name, fatal=False)
         return self._html_search_meta(name, html, display_name)
 
     def _real_extract(self, url):

From 83e7a314b4974d6d5674d88b72223d6785f29af7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 10:48:39 +0100
Subject: [PATCH 681/974] dedup AUTHORS

---
 AUTHORS | 1 -
 1 file changed, 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index 4a70e7cca..83d8161a9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -107,5 +107,4 @@ Yen Chi Hsuan
 Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
-Yen Chi Hsuan
 Paul Hartmann

From a52c6335361d0c86ddc8798d291139faa15de54c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 10:59:07 +0100
Subject: [PATCH 682/974] [cinchcast] Wrap overly long lines (#4820)

---
 youtube_dl/extractor/cinchcast.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dl/extractor/cinchcast.py
index e23c515d9..562c9bbbb 100644
--- a/youtube_dl/extractor/cinchcast.py
+++ b/youtube_dl/extractor/cinchcast.py
@@ -28,7 +28,10 @@ class CinchcastIE(InfoExtractor):
             item, './{http://developer.longtailvideo.com/trac/}date')
         upload_date = unified_strdate(date_str, day_first=False)
         # duration is present but wrong
-        formats = [{'format_id': 'main', 'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url']}]
+        formats = [{
+            'format_id': 'main',
+            'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url'],
+        }]
         backup_url = xpath_text(
             item, './{http://developer.longtailvideo.com/trac/}backupContent')
         if backup_url:

From 70fca8d694e6f611384fdcabad748fb7a65235e4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 10:59:19 +0100
Subject: [PATCH 683/974] [youtube] Remove unused import

---
 youtube_dl/extractor/youtube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a9e444351..3b3678c6e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -28,7 +28,6 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
-    js_to_json,
     OnDemandPagedList,
     orderedSet,
     unescapeHTML,

From dc35bfd2d50728d04c1b8213524f6edd2a06513c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 15:52:31 +0100
Subject: [PATCH 684/974] [test/helper] Clarify which keys have to be added

---
 test/helper.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 27a68091f..805467c82 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -148,9 +148,15 @@ def expect_info_dict(self, got_dict, expected_dict):
                 return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'").replace('\n', '\\n')
             else:
                 return repr(v)
-        info_dict_str = ''.join(
-            '    %s: %s,\n' % (_repr(k), _repr(v))
-            for k, v in test_info_dict.items())
+        info_dict_str = ''
+        if len(missing_keys) != len(expected_dict):
+            info_dict_str += ''.join(
+                '    %s: %s,\n' % (_repr(k), _repr(v))
+                for k, v in test_info_dict.items() if k not in missing_keys)
+            info_dict_str += '\n'
+        info_dict_str += ''.join(
+            '    %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
+            for k in missing_keys)
         write_string(
             '\n\'info_dict\': {\n' + info_dict_str + '}\n', out=sys.stderr)
         self.assertFalse(

From b04b8852710834bfbfdd38722e2cc212eab2a9ec Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 15:53:16 +0100
Subject: [PATCH 685/974] [extractor/common] Document all protocol values

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 478232682..8b4ef3f09 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -89,7 +89,8 @@ class InfoExtractor(object):
                     * player_url SWF Player URL (used for rtmpdump).
                     * protocol   The protocol that will be used for the actual
                                  download, lower-case.
-                                 "http", "https", "rtsp", "rtmp", "m3u8" or so.
+                                 "http", "https", "rtsp", "rtmp", "rtmpe",
+                                 "m3u8", or "m3u8_native".
                     * preference Order number of this format. If this field is
                                  present and not None, the formats get sorted
                                  by this field, regardless of all other values.

From 01c46659c4aa312a6d7c935d3a92d9fa07d66c00 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 15:53:58 +0100
Subject: [PATCH 686/974] [washingtonpost] Catch more UUIDs

---
 youtube_dl/extractor/washingtonpost.py | 34 +++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index c17bebd6e..72eb010f8 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -11,9 +12,10 @@ from ..utils import (
 
 class WashingtonPostIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
         'info_dict': {
+            'id': 'sinkhole-of-bureaucracy',
             'title': 'Sinkhole of bureaucracy',
         },
         'playlist': [{
@@ -40,15 +42,38 @@ class WashingtonPostIE(InfoExtractor):
                 'upload_date': '20140322',
                 'uploader': 'The Washington Post',
             },
+        }],
+    }, {
+        'url': 'http://www.washingtonpost.com/blogs/wonkblog/wp/2014/12/31/one-airline-figured-out-how-to-make-sure-its-airplanes-never-disappear/',
+        'info_dict': {
+            'id': 'one-airline-figured-out-how-to-make-sure-its-airplanes-never-disappear',
+            'title': 'One airline figured out how to make sure its airplanes never disappear',
+        },
+        'playlist': [{
+            'md5': 'a7c1b5634ba5e57a6a82cdffa5b1e0d0',
+            'info_dict': {
+                'id': '0e4bb54c-9065-11e4-a66f-0ca5037a597d',
+                'ext': 'mp4',
+                'description': 'Washington Post transportation reporter Ashley Halsey III explains why a plane\'s black box needs to be recovered from a crash site instead of having its information streamed in real time throughout the flight.',
+                'upload_date': '20141230',
+                'uploader': 'The Washington Post',
+                'timestamp': 1419974765,
+                'title': 'Why black boxes don’t transmit data in real time',
+            }
         }]
-    }
+    }]
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
 
         title = self._og_search_title(webpage)
-        uuids = re.findall(r'data-video-uuid="([^"]+)"', webpage)
+
+        uuids = re.findall(r'''(?x)
+            (?:
+                <div\s+class="posttv-video-embed[^>]*?data-uuid=|
+                data-video-uuid=
+            )"([^"]+)"''', webpage)
         entries = []
         for i, uuid in enumerate(uuids, start=1):
             vinfo_all = self._download_json(
@@ -75,10 +100,11 @@ class WashingtonPostIE(InfoExtractor):
                 'filesize': s.get('fileSize'),
                 'url': s.get('url'),
                 'ext': 'mp4',
+                'preference': -100 if s.get('type') == 'smil' else None,
                 'protocol': {
                     'MP4': 'http',
                     'F4F': 'f4m',
-                }.get(s.get('type'))
+                }.get(s.get('type')),
             } for s in vinfo.get('streams', [])]
             source_media_url = vinfo.get('sourceMediaURL')
             if source_media_url:

From d543bdc351c073621e2f8647fad0f9c8da1cf5b6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 16:06:55 +0100
Subject: [PATCH 687/974] [downloader/f4m] Clarify that we should eventually
 just implement the DRM scheme (#3000)

---
 youtube_dl/downloader/f4m.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 29de7630d..13055a67e 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -238,13 +238,13 @@ class F4mFD(FileDownloader):
                   doc.findall(_add_ns('drmAdditionalHeaderSet'))):
             # If id attribute is missing it's valid for all media nodes
             # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
-            if not 'id' in e.attrib:
-                self.report_error('Media is DRM protected')
+            if 'id' not in e.attrib:
+                self.report_error('Missing ID in f4m DRM')
         media = list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib and
                                       'drmAdditionalHeaderSetId' not in e.attrib,
                             media))
         if not media:
-            self.report_error('Media is DRM protected')
+            self.report_error('Unsupported DRM')
         return media
 
     def real_download(self, filename, info_dict):

From dee3f73787226e3faaf409f8c355b5f8d5dc5632 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 21:54:14 +0600
Subject: [PATCH 688/974] [spike] Modernize

---
 youtube_dl/extractor/spike.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index a3adf54e3..abba66aee 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -25,8 +25,7 @@ class SpikeIE(MTVServicesInfoExtractor):
     _MOBILE_TEMPLATE = 'http://m.spike.com/videos/video.rbml?id=%s'
 
     def _real_extract(self, url):
-        mobj = re.search(self._VALID_URL, url)
-        mobile_id = mobj.group('mobile_id')
-        if mobile_id is not None:
+        mobile_id = self._match_id(url)
+        if mobile_id:
             url = 'http://www.spike.com/video-clips/%s' % mobile_id
         return super(SpikeIE, self)._real_extract(url)

From 20b4492c718a5c3cfb2aa493134b9ba8acb0f750 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 21:54:48 +0600
Subject: [PATCH 689/974] [spike] Improve _VALID_URL

---
 youtube_dl/extractor/spike.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index abba66aee..1af2e9725 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -7,8 +7,8 @@ from .mtv import MTVServicesInfoExtractor
 
 class SpikeIE(MTVServicesInfoExtractor):
     _VALID_URL = r'''(?x)https?://
-        (www\.spike\.com/(video-clips|episodes)/.+|
-         m\.spike\.com/videos/video.rbml\?id=(?P<mobile_id>[^&]+))
+        (?:www\.spike\.com/(?:video-clips|(?:full-)?episodes)/.+|
+         m\.spike\.com/videos/video\.rbml\?id=(?P<id>[^&]+))
         '''
     _TEST = {
         'url': 'http://www.spike.com/video-clips/lhtu8m/auction-hunters-can-allen-ride-a-hundred-year-old-motorcycle',

From e525d9a3dfb03152e133b8c0ccc8a104289cf5cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 21:57:59 +0600
Subject: [PATCH 690/974] [mtv] Extract subtitles (Closes #4811)

---
 youtube_dl/extractor/mtv.py | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 22a726327..7a3b62ebe 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -2,10 +2,11 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+    compat_str,
 )
 from ..utils import (
     ExtractorError,
@@ -22,7 +23,7 @@ def _media_xml_tag(tag):
     return '{http://search.yahoo.com/mrss/}%s' % tag
 
 
-class MTVServicesInfoExtractor(InfoExtractor):
+class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
     _MOBILE_TEMPLATE = None
 
     @staticmethod
@@ -89,6 +90,28 @@ class MTVServicesInfoExtractor(InfoExtractor):
         self._sort_formats(formats)
         return formats
 
+    def _extract_subtitles(self, mdoc, mtvn_id):
+        subtitles = {}
+        FORMATS = {
+            'scc': 'cea-608',
+            'eia-608': 'cea-608',
+            'xml': 'ttml',
+        }
+        subtitles_format = FORMATS.get(
+            self._downloader.params.get('subtitlesformat'), 'ttml')
+        for transcript in mdoc.findall('.//transcript'):
+            if transcript.get('kind') != 'captions':
+                continue
+            lang = transcript.get('srclang')
+            for typographic in transcript.findall('./typographic'):
+                captions_format = typographic.get('format')
+                if captions_format == subtitles_format:
+                    subtitles[lang] = compat_str(typographic.get('src'))
+                    break
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(mtvn_id, subtitles)
+        return self.extract_subtitles(mtvn_id, subtitles)
+
     def _get_video_info(self, itemdoc):
         uri = itemdoc.find('guid').text
         video_id = self._id_from_uri(uri)
@@ -135,6 +158,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         return {
             'title': title,
             'formats': self._extract_video_formats(mediagen_doc, mtvn_id),
+            'subtitles': self._extract_subtitles(mediagen_doc, mtvn_id),
             'id': video_id,
             'thumbnail': self._get_thumbnail_url(uri, itemdoc),
             'description': description,
@@ -167,7 +191,11 @@ class MTVServicesInfoExtractor(InfoExtractor):
             mgid = self._search_regex(
                 [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
                 webpage, 'mgid')
-        return self._get_videos_info(mgid)
+
+        videos_info = self._get_videos_info(mgid)
+        if self._downloader.params.get('listsubtitles', False):
+            return
+        return videos_info
 
 
 class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):

From ef1269fb07833a06a15d0601e54d58e1678bc06a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 22:42:11 +0600
Subject: [PATCH 691/974] [drtv] Improve _VALID_URL (#4814)

---
 youtube_dl/extractor/drtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 510ef04b0..8d113a5e8 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -6,7 +6,7 @@ from ..utils import parse_iso8601
 
 
 class DRTVIE(SubtitlesInfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)+(?P<id>[\da-z-]+)(?:[/#?]|$)'
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
     _TEST = {
         'url': 'http://www.dr.dk/tv/se/partiets-mand/partiets-mand-7-8',

From 2c58674e0ede5d656cb559a18a3b86ffbb21b3a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 22:46:53 +0600
Subject: [PATCH 692/974] [nhl] Improve _VALID_URL (#4805)

---
 youtube_dl/extractor/nhl.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index d3a4fc513..cb279074e 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -54,7 +54,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
 class NHLIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
 
     _TESTS = [{
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
@@ -92,6 +92,9 @@ class NHLIE(NHLBaseInfoExtractor):
     }, {
         'url': 'http://video.flames.nhl.com/videocenter/console?id=630616',
         'only_matching': True,
+    }, {
+        'url': 'http://video.nhl.com/videocenter/?id=736722',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From e4c17d7274235ec880857d7691bc7c818e2c7a01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:12:27 +0600
Subject: [PATCH 693/974] [nhl:news] Add extractor (Closes #4805)

---
 youtube_dl/extractor/__init__.py |  6 ++++-
 youtube_dl/extractor/nhl.py      | 41 +++++++++++++++++++++++++++-----
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d3f51d8c4..9c14b096f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -294,7 +294,11 @@ from .nextmedia import (
 )
 from .nfb import NFBIE
 from .nfl import NFLIE
-from .nhl import NHLIE, NHLVideocenterIE
+from .nhl import (
+    NHLIE,
+    NHLNewsIE,
+    NHLVideocenterIE,
+)
 from .niconico import NiconicoIE, NiconicoPlaylistIE
 from .ninegag import NineGagIE
 from .noco import NocoIE
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index cb279074e..407465998 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -20,6 +20,12 @@ class NHLBaseInfoExtractor(InfoExtractor):
     def _fix_json(json_string):
         return json_string.replace('\\\'', '\'')
 
+    def _real_extract_video(self, video_id):
+        json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
+        data = self._download_json(
+            json_url, video_id, transform_source=self._fix_json)
+        return self._extract_video(data[0])
+
     def _extract_video(self, info):
         video_id = info['id']
         self.report_extraction(video_id)
@@ -98,12 +104,35 @@ class NHLIE(NHLBaseInfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
-        data = self._download_json(
-            json_url, video_id, transform_source=self._fix_json)
-        return self._extract_video(data[0])
+        video_id = self._match_id(url)
+        return self._real_extract_video(video_id)
+
+
+class NHLNewsIE(NHLBaseInfoExtractor):
+    IE_NAME = 'nhl.com:news'
+    IE_DESC = 'NHL news'
+    _VALID_URL = r'https?://(?:www\.)?nhl\.com/ice/news\.html?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+
+    _TEST = {
+        'url': 'http://www.nhl.com/ice/news.htm?id=750727',
+        'md5': '4b3d1262e177687a3009937bd9ec0be8',
+        'info_dict': {
+            'id': '736722',
+            'ext': 'mp4',
+            'title': 'Cal Clutterbuck has been fined $2,000',
+            'description': 'md5:45fe547d30edab88b23e0dd0ab1ed9e6',
+            'duration': 37,
+            'upload_date': '20150128',
+        },
+    }
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        webpage = self._download_webpage(url, news_id)
+        video_id = self._search_regex(
+            [r'pVid(\d+)', r"nlid\s*:\s*'(\d+)'"],
+            webpage, 'video id')
+        return self._real_extract_video(video_id)
 
 
 class NHLVideocenterIE(NHLBaseInfoExtractor):

From b80505a40938444c6f7b0b80c445cc0726c39bb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:21:44 +0600
Subject: [PATCH 694/974] [mixcloud] Fix extraction (Closes #4784)

---
 youtube_dl/extractor/mixcloud.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 07d194562..162046fe6 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -10,6 +10,7 @@ from ..utils import (
     ExtractorError,
     HEADRequest,
     int_or_none,
+    str_to_int,
     parse_iso8601,
 )
 
@@ -85,15 +86,17 @@ class MixcloudIE(InfoExtractor):
         uploader_id = self._search_regex(
             r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
         description = self._og_search_description(webpage)
-        like_count = int_or_none(self._search_regex(
-            r'<meta itemprop="interactionCount" content="UserLikes:([0-9]+)"',
+        like_count = str_to_int(self._search_regex(
+            [r'<meta itemprop="interactionCount" content="UserLikes:([0-9]+)"',
+             r'/favorites/?">([0-9]+)<'],
             webpage, 'like count', fatal=False))
-        view_count = int_or_none(self._search_regex(
-            r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
+        view_count = str_to_int(self._search_regex(
+            [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
+             r'/listeners/?">([0-9,.]+)</a>'],
             webpage, 'play count', fatal=False))
         timestamp = parse_iso8601(self._search_regex(
             r'<time itemprop="dateCreated" datetime="([^"]+)">',
-            webpage, 'upload date'))
+            webpage, 'upload date', default=None))
 
         return {
             'id': track_id,

From 439d9be27dc3cf811f3337e40c3222bc8d2ed339 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:21:58 +0600
Subject: [PATCH 695/974] [mixcloud] Remove unused import

---
 youtube_dl/extractor/mixcloud.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 162046fe6..141bb463e 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -9,7 +9,6 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     HEADRequest,
-    int_or_none,
     str_to_int,
     parse_iso8601,
 )

From 0692ef86ef85da69074b00b5525c13bfde9159de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:47:09 +0600
Subject: [PATCH 696/974] [bbccouk] Improve _VALID_URL

---
 youtube_dl/extractor/bbccouk.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 1cf48fe0d..126c8824c 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -10,7 +10,7 @@ from ..compat import compat_HTTPError
 class BBCCoUkIE(SubtitlesInfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
     _TESTS = [
         {
@@ -118,6 +118,9 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         }, {
             'url': 'http://www.bbc.co.uk/music/clips#p02frcc3',
             'only_matching': True,
+        }, {
+            'url': 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo',
+            'only_matching': True,
         }
     ]
 

From 91755ee384ddee314ba40748a44ad1289df8b86c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 19:43:46 +0100
Subject: [PATCH 697/974] [comedycentral:shows] Generate better IDs and add a
 test for that

---
 youtube_dl/extractor/comedycentral.py | 34 ++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 15ca361f0..f9256dcf4 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -49,7 +49,9 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
                               |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))
                           )|
                           (?P<interview>
-                              extended-interviews/(?P<interID>[0-9a-z]+)/(?:playlist_tds_extended_)?(?P<interview_title>.*?)(/.*?)?)))
+                              extended-interviews/(?P<interID>[0-9a-z]+)/
+                              (?:playlist_tds_extended_)?(?P<interview_title>[^/?#]*?)
+                              (?:/[^/?#]?|[?#]|$))))
                      '''
     _TESTS = [{
         'url': 'http://thedailyshow.cc.com/watch/thu-december-13-2012/kristen-stewart',
@@ -62,6 +64,35 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'uploader': 'thedailyshow',
             'title': 'thedailyshow kristen-stewart part 1',
         }
+    }, {
+        'url': 'http://thedailyshow.cc.com/extended-interviews/b6364d/sarah-chayes-extended-interview',
+        'info_dict': {
+            'id': 'sarah-chayes-extended-interview',
+            'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+            'title': 'thedailyshow Sarah Chayes Extended Interview',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '0baad492-cbec-4ec1-9e50-ad91c291127f',
+                'ext': 'mp4',
+                'upload_date': '20150129',
+                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                'uploader': 'thedailyshow',
+                'title': 'thedailyshow sarah-chayes-extended-interview part 1',
+            }
+         }, {
+            'info_dict': {
+                'id': '1e4fb91b-8ce7-4277-bd7c-98c9f1bbd283',
+                'ext': 'mp4',
+                'upload_date': '20150129',
+                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                'uploader': 'thedailyshow',
+                'title': 'thedailyshow sarah-chayes-extended-interview part 2',
+            }
+        }],
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://thedailyshow.cc.com/extended-interviews/xm3fnq/andrew-napolitano-extended-interview',
         'only_matching': True,
@@ -230,6 +261,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
 
         return {
             '_type': 'playlist',
+            'id': epTitle,
             'entries': entries,
             'title': show_name + ' ' + title,
             'description': description,

From 6d593c3276da6a0a1ad738c779b87a1998c5b06b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Jan 2015 03:50:11 +0600
Subject: [PATCH 698/974] [YoutubeDL] Fix video+audio format_id (Closes #4824)

---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ce3abaee3..e5a96cad5 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1074,7 +1074,8 @@ class YoutubeDL(object):
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
-                                'format_id': rf,
+                                'format_id': '%s+%s' % (formats_info[0].get('format_id'),
+                                                        formats_info[1].get('format_id')),
                                 'width': formats_info[0].get('width'),
                                 'height': formats_info[0].get('height'),
                                 'resolution': formats_info[0].get('resolution'),

From cc57bd33a803a061ac0bbe971088fb1ef78be82b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 30 Jan 2015 23:05:06 +0100
Subject: [PATCH 699/974] [rtve] Fix extraction

Skip live stream test, we can't use the f4m manifest yet
---
 youtube_dl/extractor/rtve.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 0ce22d60c..9e8688fbf 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -74,7 +74,8 @@ class RTVEALaCartaIE(InfoExtractor):
             'id': '1694255',
             'ext': 'flv',
             'title': 'TODO',
-        }
+        },
+        'skip': 'The f4m manifest can\'t be used yet',
     }]
 
     def _real_extract(self, url):
@@ -86,6 +87,18 @@ class RTVEALaCartaIE(InfoExtractor):
         png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % video_id
         png = self._download_webpage(png_url, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
+        if not video_url.endswith('.f4m'):
+            auth_url = video_url.replace(
+                'resources/', 'auth/resources/'
+            ).replace('.net.rtve', '.multimedia.cdn.rtve')
+            video_path = self._download_webpage(
+                auth_url, video_id, 'Getting video url')
+            # Use mvod.akcdn instead of flash.akamaihd.multimedia.cdn to get
+            # the right Content-Length header and the mp4 format
+            video_url = (
+                'http://mvod.akcdn.rtve.es/{0}&v=2.6.8'
+                '&fp=MAC%2016,0,0,296&r=MRUGG&g=OEOJWFXNFGCP'.format(video_path)
+            )
 
         return {
             'id': video_id,

From cd596028d6c873ac7e9a61902bbe8f86d517b5b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 30 Jan 2015 23:12:53 +0100
Subject: [PATCH 700/974] [rtve] Recognize mobile urls (fixes #4823)

---
 youtube_dl/extractor/rtve.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 9e8688fbf..3469d9578 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -57,7 +57,7 @@ def _decrypt_url(png):
 class RTVEALaCartaIE(InfoExtractor):
     IE_NAME = 'rtve.es:alacarta'
     IE_DESC = 'RTVE a la carta'
-    _VALID_URL = r'http://www\.rtve\.es/alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'http://www\.rtve\.es/(m/)?alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
@@ -76,6 +76,9 @@ class RTVEALaCartaIE(InfoExtractor):
             'title': 'TODO',
         },
         'skip': 'The f4m manifest can\'t be used yet',
+    }, {
+        'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From e41b1f73854d8a422235a3b92ca4255affbcd39e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 31 Jan 2015 10:51:39 +0100
Subject: [PATCH 701/974] Fix flake8 errors

---
 youtube_dl/downloader/f4m.py          |  2 +-
 youtube_dl/extractor/comedycentral.py | 41 ++++++++++++++-------------
 youtube_dl/extractor/spike.py         |  2 --
 3 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 13055a67e..0e7a1c200 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -231,7 +231,7 @@ class F4mFD(FileDownloader):
     """
 
     def _get_unencrypted_media(self, doc):
-        media=doc.findall(_add_ns('media'))
+        media = doc.findall(_add_ns('media'))
         if not media:
             self.report_error('No media found')
         for e in (doc.findall(_add_ns('drmAdditionalHeader')) +
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index f9256dcf4..b24538981 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -71,25 +71,28 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
             'title': 'thedailyshow Sarah Chayes Extended Interview',
         },
-        'playlist': [{
-            'info_dict': {
-                'id': '0baad492-cbec-4ec1-9e50-ad91c291127f',
-                'ext': 'mp4',
-                'upload_date': '20150129',
-                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
-                'uploader': 'thedailyshow',
-                'title': 'thedailyshow sarah-chayes-extended-interview part 1',
-            }
-         }, {
-            'info_dict': {
-                'id': '1e4fb91b-8ce7-4277-bd7c-98c9f1bbd283',
-                'ext': 'mp4',
-                'upload_date': '20150129',
-                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
-                'uploader': 'thedailyshow',
-                'title': 'thedailyshow sarah-chayes-extended-interview part 2',
-            }
-        }],
+        'playlist': [
+            {
+                'info_dict': {
+                    'id': '0baad492-cbec-4ec1-9e50-ad91c291127f',
+                    'ext': 'mp4',
+                    'upload_date': '20150129',
+                    'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                    'uploader': 'thedailyshow',
+                    'title': 'thedailyshow sarah-chayes-extended-interview part 1',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': '1e4fb91b-8ce7-4277-bd7c-98c9f1bbd283',
+                    'ext': 'mp4',
+                    'upload_date': '20150129',
+                    'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                    'uploader': 'thedailyshow',
+                    'title': 'thedailyshow sarah-chayes-extended-interview part 2',
+                },
+            },
+        ],
         'params': {
             'skip_download': True,
         },
diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index 1af2e9725..e529bb55c 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .mtv import MTVServicesInfoExtractor
 
 

From e77d2975afda3750104c90b7c0012c02a2a00b10 Mon Sep 17 00:00:00 2001
From: light94 <mishra.rahul1712@gmail.com>
Date: Sun, 1 Feb 2015 00:10:58 +0530
Subject: [PATCH 702/974] Handling Connection Reset by Peer Error

---
 youtube_dl/downloader/http.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 8a1d578d5..9b023c5c1 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -3,6 +3,9 @@ from __future__ import unicode_literals
 import os
 import time
 
+from socket import error as SocketError
+import errno
+
 from .common import FileDownloader
 from ..compat import (
     compat_urllib_request,
@@ -99,6 +102,11 @@ class HttpFD(FileDownloader):
                             resume_len = 0
                             open_mode = 'wb'
                             break
+
+            except SocketError as e:
+                if e.errno != errno.ECONNRESET:
+                    raise # Not error we are looking for
+                pass
             # Retry
             count += 1
             if count <= retries:

From 1ee94db2d03770c0280047c974f0e6886d2a753f Mon Sep 17 00:00:00 2001
From: kinetoskombi <filipeamiranda@gmail.com>
Date: Sat, 31 Jan 2015 20:07:43 -0200
Subject: [PATCH 703/974] [globo] Fix error on some globo videos

---
 youtube_dl/extractor/globo.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 6949a57c7..d936be384 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -70,6 +70,19 @@ class GloboIE(InfoExtractor):
                 'like_count': int,
             }
         },
+        {
+            'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
+            'md5': 'c1defca721ce25b2354e927d3e4b3dec',
+            'info_dict': {
+                'id': '3928201',
+                'ext': 'mp4',
+                'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
+                'duration': 1472.906,
+                'uploader': 'Canal Brasil',
+                'uploader_id': 705,
+                'like_count': int,
+            }
+        },
     ]
 
     class MD5():
@@ -355,7 +368,9 @@ class GloboIE(InfoExtractor):
 
         for resource in video['resources']:
             resource_id = resource.get('_id')
-            if not resource_id:
+            resource_height = resource.get('height')
+
+            if not (resource_id or resource_height):
                 continue
 
             security = self._download_json(
@@ -384,7 +399,7 @@ class GloboIE(InfoExtractor):
             formats.append({
                 'url': '%s?h=%s&k=%s' % (resource['url'], signed_hash, 'flash'),
                 'format_id': resource_id,
-                'height': resource['height']
+                'height': resource_height
             })
 
         self._sort_formats(formats)

From 8c72beb25eaa1f1960a3b50295bcdcfd50c450af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 04:36:24 +0600
Subject: [PATCH 704/974] [globo] Properly extract m3u8 formats (#4346 #4832)

---
 youtube_dl/extractor/globo.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index d936be384..29638a194 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -368,9 +368,7 @@ class GloboIE(InfoExtractor):
 
         for resource in video['resources']:
             resource_id = resource.get('_id')
-            resource_height = resource.get('height')
-
-            if not (resource_id or resource_height):
+            if not resource_id:
                 continue
 
             security = self._download_json(
@@ -396,11 +394,16 @@ class GloboIE(InfoExtractor):
             signed_md5 = self.MD5.b64_md5(received_md5 + compat_str(sign_time) + padding)
             signed_hash = hash_code + compat_str(received_time) + received_random + compat_str(sign_time) + padding + signed_md5
 
-            formats.append({
-                'url': '%s?h=%s&k=%s' % (resource['url'], signed_hash, 'flash'),
-                'format_id': resource_id,
-                'height': resource_height
-            })
+            resource_url = resource['url']
+            signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
+            if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(signed_url, resource_id, 'mp4'))
+            else:
+                formats.append({
+                    'url': signed_url,
+                    'format_id': resource_id,
+                    'height': resource.get('height'),
+                })
 
         self._sort_formats(formats)
 

From 8f9312c38779f4e0746c01ef9a8a93ab4767699b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 11:30:56 +0100
Subject: [PATCH 705/974] Appease pyflakes8-3

---
 youtube_dl/YoutubeDL.py |  3 ++-
 youtube_dl/compat.py    | 37 +++++++++++++++++++++----------------
 youtube_dl/utils.py     |  7 ++++---
 3 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e5a96cad5..c18ce9660 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -25,6 +25,7 @@ if os.name == 'nt':
     import ctypes
 
 from .compat import (
+    compat_basestring,
     compat_cookiejar,
     compat_expanduser,
     compat_http_client,
@@ -1558,7 +1559,7 @@ class YoutubeDL(object):
         # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
         # To work around aforementioned issue we will replace request's original URL with
         # percent-encoded one
-        req_is_string = isinstance(req, basestring if sys.version_info < (3, 0) else compat_str)
+        req_is_string = isinstance(req, compat_basestring)
         url = req if req_is_string else req.get_full_url()
         url_escaped = escape_url(url)
 
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 497ca52de..ec685d50e 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -114,6 +114,26 @@ except ImportError:
             string += pct_sequence.decode(encoding, errors)
         return string
 
+try:
+    compat_str = unicode  # Python 2
+except NameError:
+    compat_str = str
+
+try:
+    compat_basestr = basestring  # Python 2
+except NameError:
+    compat_basestr = str
+
+try:
+    compat_chr = unichr  # Python 2
+except NameError:
+    compat_chr = chr
+
+try:
+    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
+except ImportError:  # Python 2.6
+    from xml.parsers.expat import ExpatError as compat_xml_parse_error
+
 
 try:
     from urllib.parse import parse_qs as compat_parse_qs
@@ -123,7 +143,7 @@ except ImportError:  # Python 2
 
     def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
                    encoding='utf-8', errors='replace'):
-        qs, _coerce_result = qs, unicode
+        qs, _coerce_result = qs, compat_str
         pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
         r = []
         for name_value in pairs:
@@ -162,21 +182,6 @@ except ImportError:  # Python 2
                 parsed_result[name] = [value]
         return parsed_result
 
-try:
-    compat_str = unicode  # Python 2
-except NameError:
-    compat_str = str
-
-try:
-    compat_chr = unichr  # Python 2
-except NameError:
-    compat_chr = chr
-
-try:
-    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
-except ImportError:  # Python 2.6
-    from xml.parsers.expat import ExpatError as compat_xml_parse_error
-
 try:
     from shlex import quote as shlex_quote
 except ImportError:  # Python < 3.3
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a4c9813ec..4ade0554e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -32,6 +32,7 @@ import xml.etree.ElementTree
 import zlib
 
 from .compat import (
+    compat_basestring,
     compat_chr,
     compat_getenv,
     compat_html_entities,
@@ -140,7 +141,7 @@ else:
     def find_xpath_attr(node, xpath, key, val):
         # Here comes the crazy part: In 2.6, if the xpath is a unicode,
         # .//node does not match if a node is a direct child of . !
-        if isinstance(xpath, unicode):
+        if isinstance(xpath, compat_str):
             xpath = xpath.encode('ascii')
 
         for f in node.findall(xpath):
@@ -1262,7 +1263,7 @@ def float_or_none(v, scale=1, invscale=1, default=None):
 
 
 def parse_duration(s):
-    if not isinstance(s, basestring if sys.version_info < (3, 0) else compat_str):
+    if not isinstance(s, compat_basestring):
         return None
 
     s = s.strip()
@@ -1426,7 +1427,7 @@ def uppercase_escape(s):
 
 def escape_rfc3986(s):
     """Escape non-ASCII characters as suggested by RFC 3986"""
-    if sys.version_info < (3, 0) and isinstance(s, unicode):
+    if sys.version_info < (3, 0) and isinstance(s, compat_str):
         s = s.encode('utf-8')
     return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
 

From 0196149c5bf6b72c1d268a4beb2c50e4ecb063d7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 11:36:59 +0100
Subject: [PATCH 706/974] [compat] Correct compat_basestring definition

---
 youtube_dl/compat.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index ec685d50e..e989cdbbd 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -120,9 +120,9 @@ except NameError:
     compat_str = str
 
 try:
-    compat_basestr = basestring  # Python 2
+    compat_basestring = basestring  # Python 2
 except NameError:
-    compat_basestr = str
+    compat_basestring = str
 
 try:
     compat_chr = unichr  # Python 2
@@ -367,6 +367,7 @@ def workaround_optparse_bug9161():
 
 __all__ = [
     'compat_HTTPError',
+    'compat_basestring',
     'compat_chr',
     'compat_cookiejar',
     'compat_expanduser',

From 9158b2b301f83a1cec13a6340f354646434c69c1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 11:58:37 +0100
Subject: [PATCH 707/974] [mpora] Modernize

---
 youtube_dl/extractor/mpora.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/mpora.py b/youtube_dl/extractor/mpora.py
index 88c9501cd..6db3c67a5 100644
--- a/youtube_dl/extractor/mpora.py
+++ b/youtube_dl/extractor/mpora.py
@@ -1,21 +1,19 @@
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import int_or_none
 
 
 class MporaIE(InfoExtractor):
-    _VALID_URL = r'^https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
+    _VALID_URL = r'https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
     IE_NAME = 'MPORA'
 
     _TEST = {
         'url': 'http://mpora.de/videos/AAdo8okx4wiz/embed?locale=de',
-        'file': 'AAdo8okx4wiz.mp4',
         'md5': 'a7a228473eedd3be741397cf452932eb',
         'info_dict': {
+            'id': 'AAdo8okx4wiz',
+            'ext': 'mp4',
             'title': 'Katy Curd -  Winter in the Forest',
             'duration': 416,
             'uploader': 'Peter Newman Media',
@@ -23,14 +21,12 @@ class MporaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         data_json = self._search_regex(
             r"new FM\.Player\('[^']+',\s*(\{.*?)\).player;", webpage, 'json')
-
-        data = json.loads(data_json)
+        data = self._parse_json(data_json, video_id)
 
         uploader = data['info_overlay'].get('username')
         duration = data['video']['duration'] // 1000

From 29aef5a33c46ceaa5884a3f074fbb5af4fa44f32 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:00:47 +0100
Subject: [PATCH 708/974] [ard] Remove deleted video test case

---
 youtube_dl/extractor/ard.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 967bd865c..783b53e23 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -23,13 +23,7 @@ class ARDMediathekIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
-        'file': '22429276.mp4',
-        'md5': '469751912f1de0816a9fc9df8336476c',
-        'info_dict': {
-            'title': 'Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?',
-            'description': 'Das Erste Mediathek [ARD]: Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?, Anne Will, Über die Spionage-Affäre diskutieren Clemens Binninger, Katrin Göring-Eckardt, Georg Mascolo, Andrew B. Denison und Constanze Kurz.. Das Video zur Sendung Anne Will am Mittwoch, 16.07.2014',
-        },
-        'skip': 'Blocked outside of Germany',
+        'only_matching': True,
     }, {
         'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916',
         'info_dict': {

From ca0f500ecfde0a0ada7002348e8b04ac5c79d4b4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:08:21 +0100
Subject: [PATCH 709/974] [mtv] Modernize and clean up test

---
 youtube_dl/extractor/mtv.py | 34 +++++++++++++---------------------
 1 file changed, 13 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 7a3b62ebe..bc7f49ebb 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -79,12 +79,15 @@ class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
             try:
                 _, _, ext = rendition.attrib['type'].partition('/')
                 rtmp_video_url = rendition.find('./src').text
-                formats.append({'ext': ext,
-                                'url': self._transform_rtmp_url(rtmp_video_url),
-                                'format_id': rendition.get('bitrate'),
-                                'width': int(rendition.get('width')),
-                                'height': int(rendition.get('height')),
-                                })
+                if rtmp_video_url.endswith('siteunavail.png'):
+                    continue
+                formats.append({
+                    'ext': ext,
+                    'url': self._transform_rtmp_url(rtmp_video_url),
+                    'format_id': rendition.get('bitrate'),
+                    'width': int(rendition.get('width')),
+                    'height': int(rendition.get('height')),
+                })
             except (KeyError, TypeError):
                 raise ExtractorError('Invalid rendition field.')
         self._sort_formats(formats)
@@ -240,25 +243,14 @@ class MTVIE(MTVServicesInfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.mtv.com/videos/misc/853555/ours-vh1-storytellers.jhtml',
-            'file': '853555.mp4',
             'md5': '850f3f143316b1e71fa56a4edfd6e0f8',
             'info_dict': {
+                'id': '853555',
+                'ext': 'mp4',
                 'title': 'Taylor Swift - "Ours (VH1 Storytellers)"',
                 'description': 'Album: Taylor Swift performs "Ours" for VH1 Storytellers at Harvey Mudd College.',
             },
         },
-        {
-            'add_ie': ['Vevo'],
-            'url': 'http://www.mtv.com/videos/taylor-swift/916187/everything-has-changed-ft-ed-sheeran.jhtml',
-            'file': 'USCJY1331283.mp4',
-            'md5': '73b4e7fcadd88929292fe52c3ced8caf',
-            'info_dict': {
-                'title': 'Everything Has Changed',
-                'upload_date': '20130606',
-                'uploader': 'Taylor Swift',
-            },
-            'skip': 'VEVO is only available in some countries',
-        },
     ]
 
     def _get_thumbnail_url(self, uri, itemdoc):
@@ -272,8 +264,8 @@ class MTVIE(MTVServicesInfoExtractor):
             webpage = self._download_webpage(url, video_id)
 
             # Some videos come from Vevo.com
-            m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";',
-                               webpage, re.DOTALL)
+            m_vevo = re.search(
+                r'(?s)isVevoVideo = true;.*?vevoVideoId = "(.*?)";', webpage)
             if m_vevo:
                 vevo_id = m_vevo.group(1)
                 self.to_screen('Vevo video detected: %s' % vevo_id)

From 2886be15aa00e064c494eb0b6977246cdb905b1f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:10:15 +0100
Subject: [PATCH 710/974] [defense] Modernize

---
 youtube_dl/extractor/defense.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index 5e50c63d9..1637fa4a7 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -8,33 +8,34 @@ from .common import InfoExtractor
 
 class DefenseGouvFrIE(InfoExtractor):
     IE_NAME = 'defense.gouv.fr'
-    _VALID_URL = (r'http://.*?\.defense\.gouv\.fr/layout/set/'
-                  r'ligthboxvideo/base-de-medias/webtv/(.*)')
+    _VALID_URL = r'http://.*?\.defense\.gouv\.fr/layout/set/ligthboxvideo/base-de-medias/webtv/(?P<id>[^/?#]*)'
 
     _TEST = {
         'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
-        'file': '11213.mp4',
         'md5': '75bba6124da7e63d2d60b5244ec9430c',
-        "info_dict": {
-            "title": "attaque-chimique-syrienne-du-21-aout-2013-1"
+        'info_dict': {
+            'id': '11213',
+            'ext': 'mp4',
+            'title': 'attaque-chimique-syrienne-du-21-aout-2013-1'
         }
     }
 
     def _real_extract(self, url):
-        title = re.match(self._VALID_URL, url).group(1)
+        title = self._match_id(url)
         webpage = self._download_webpage(url, title)
+
         video_id = self._search_regex(
             r"flashvars.pvg_id=\"(\d+)\";",
             webpage, 'ID')
 
         json_url = ('http://static.videos.gouv.fr/brightcovehub/export/json/'
                     + video_id)
-        info = self._download_webpage(json_url, title,
-                                      'Downloading JSON config')
-        video_url = json.loads(info)['renditions'][0]['url']
+        info = self._download_json(json_url, title, 'Downloading JSON config')
+        video_url = info['renditions'][0]['url']
 
-        return {'id': video_id,
-                'ext': 'mp4',
-                'url': video_url,
-                'title': title,
-                }
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'url': video_url,
+            'title': title,
+        }

From 96aded8d3d48c5987a687b9233351ccbfc9c9237 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:11:14 +0100
Subject: [PATCH 711/974] [rottentomatoes] Modernize test definition

---
 youtube_dl/extractor/rottentomatoes.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rottentomatoes.py b/youtube_dl/extractor/rottentomatoes.py
index c1500b82f..e8bb20a08 100644
--- a/youtube_dl/extractor/rottentomatoes.py
+++ b/youtube_dl/extractor/rottentomatoes.py
@@ -10,8 +10,9 @@ class RottenTomatoesIE(VideoDetectiveIE):
 
     _TEST = {
         'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers/11028566/',
-        'file': '613340.mp4',
         'info_dict': {
+            'id': '613340',
+            'ext': 'mp4',
             'title': 'TOY STORY 3',
             'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.',
         },

From 146303136fa30ef0edf2dfbac5acd69fb2f0dbcc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:11:20 +0100
Subject: [PATCH 712/974] [nerdcubed] Modernize test definition

---
 youtube_dl/extractor/nerdcubed.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nerdcubed.py b/youtube_dl/extractor/nerdcubed.py
index efc903afa..dff78e486 100644
--- a/youtube_dl/extractor/nerdcubed.py
+++ b/youtube_dl/extractor/nerdcubed.py
@@ -11,6 +11,7 @@ class NerdCubedFeedIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.nerdcubed.co.uk/feed.json',
         'info_dict': {
+            'id': 'nerdcubed-feed',
             'title': 'nerdcubed.co.uk feed',
         },
         'playlist_mincount': 1300,

From fc09240e24c2455fa9373f77a13bf4649b90ca87 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:12:27 +0100
Subject: [PATCH 713/974] [vimeo] Modernize test definition

---
 youtube_dl/extractor/vimeo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 06b0bed41..1bb473514 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -501,9 +501,10 @@ class VimeoReviewIE(InfoExtractor):
     _VALID_URL = r'https?://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
-        'file': '75524534.mp4',
         'md5': 'c507a72f780cacc12b2248bb4006d253',
         'info_dict': {
+            'id': '75524534',
+            'ext': 'mp4',
             'title': "DICK HARDWICK 'Comedian'",
             'uploader': 'Richard Hardwick',
         }

From 17d2712d9c58af788f234a2682d486f1d0a3ab78 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:00:54 +0100
Subject: [PATCH 714/974] [teamcoco] Modernize and fix extraction

---
 youtube_dl/extractor/teamcoco.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 5fa67eb8d..18a823719 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -10,17 +10,19 @@ class TeamcocoIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://teamcoco.com/video/80187/conan-becomes-a-mary-kay-beauty-consultant',
-            'file': '80187.mp4',
             'md5': '3f7746aa0dc86de18df7539903d399ea',
             'info_dict': {
+                'id': '80187',
+                'ext': 'mp4',
                 'title': 'Conan Becomes A Mary Kay Beauty Consultant',
                 'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.'
             }
         }, {
             'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
-            'file': '19705.mp4',
             'md5': 'cde9ba0fa3506f5f017ce11ead928f9a',
             'info_dict': {
+                'id': '19705',
+                'ext': 'mp4',
                 "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
                 "title": "Louis C.K. Interview Pt. 1 11/3/11"
             }
@@ -36,7 +38,7 @@ class TeamcocoIE(InfoExtractor):
         video_id = mobj.group("video_id")
         if not video_id:
             video_id = self._html_search_regex(
-                r'data-node-id="(\d+?)"',
+                r'<div\s+class="player".*?data-id="(\d+?)"',
                 webpage, 'video id')
 
         data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id

From 057c0609fc6620017ff4a5b30847e0dc2644df9f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:01:33 +0100
Subject: [PATCH 715/974] [toutv] Modernize test definition

---
 youtube_dl/extractor/toutv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 2837f9c8e..4797d1310 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -16,8 +16,9 @@ class TouTvIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.tou.tv/30-vies/S04E41',
-        'file': '30-vies_S04E41.mp4',
         'info_dict': {
+            'id': '30-vies_S04E41',
+            'ext': 'mp4',
             'title': '30 vies Saison 4 / Épisode 41',
             'description': 'md5:da363002db82ccbe4dafeb9cab039b09',
             'age_limit': 8,

From 228d30ed0616f6c1b5cff335b8b56f4fde48b76d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:03:03 +0100
Subject: [PATCH 716/974] [la7] Modernize

---
 youtube_dl/extractor/la7.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/la7.py b/youtube_dl/extractor/la7.py
index db2028e9f..60625c42d 100644
--- a/youtube_dl/extractor/la7.py
+++ b/youtube_dl/extractor/la7.py
@@ -20,9 +20,10 @@ class LA7IE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.la7.tv/richplayer/?assetid=50355319',
-        'file': '50355319.mp4',
         'md5': 'ec7d1f0224d20ba293ab56cf2259651f',
         'info_dict': {
+            'id': '50355319',
+            'ext': 'mp4',
             'title': 'IL DIVO',
             'description': 'Un film di Paolo Sorrentino con Toni Servillo, Anna Bonaiuto, Giulio Bosetti  e Flavio Bucci',
             'duration': 6254,
@@ -31,9 +32,7 @@ class LA7IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         xml_url = 'http://www.la7.tv/repliche/content/index.php?contentId=%s' % video_id
         doc = self._download_xml(xml_url, video_id)
 

From 9a3e5e69554c06fbf8bb7fd248c2e921593f8046 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:03:55 +0100
Subject: [PATCH 717/974] [kankan] Modernize

---
 youtube_dl/extractor/kankan.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kankan.py b/youtube_dl/extractor/kankan.py
index dbfe4cc03..364dc878e 100644
--- a/youtube_dl/extractor/kankan.py
+++ b/youtube_dl/extractor/kankan.py
@@ -13,17 +13,17 @@ class KankanIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://yinyue.kankan.com/vod/48/48863.shtml',
-        'file': '48863.flv',
         'md5': '29aca1e47ae68fc28804aca89f29507e',
         'info_dict': {
+            'id': '48863',
+            'ext': 'flv',
             'title': 'Ready To Go',
         },
         'skip': 'Only available from China',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._search_regex(r'(?:G_TITLE=|G_MOVIE_TITLE = )[\'"](.+?)[\'"]', webpage, 'video title')

From c4e817ce4aeeb4012b0e128afe6c138d3d905a24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:06:55 +0100
Subject: [PATCH 718/974] [france2.fr:generation-quoi] Modernize

---
 youtube_dl/extractor/francetv.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index bbc760a49..170d68075 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -230,12 +230,13 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
 
 class GenerationQuoiIE(InfoExtractor):
     IE_NAME = 'france2.fr:generation-quoi'
-    _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<name>.*)(\?|$)'
+    _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<id>[^/?#]+)'
 
     _TEST = {
         'url': 'http://generation-quoi.france2.fr/portrait/garde-a-vous',
-        'file': 'k7FJX8VBcvvLmX4wA5Q.mp4',
         'info_dict': {
+            'id': 'k7FJX8VBcvvLmX4wA5Q',
+            'ext': 'mp4',
             'title': 'Génération Quoi - Garde à Vous',
             'uploader': 'Génération Quoi',
         },
@@ -243,14 +244,12 @@ class GenerationQuoiIE(InfoExtractor):
             # It uses Dailymotion
             'skip_download': True,
         },
-        'skip': 'Only available from France',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
-        info_url = compat_urlparse.urljoin(url, '/medias/video/%s.json' % name)
-        info_json = self._download_webpage(info_url, name)
+        display_id = self._match_id(url)
+        info_url = compat_urlparse.urljoin(url, '/medias/video/%s.json' % display_id)
+        info_json = self._download_webpage(info_url, display_id)
         info = json.loads(info_json)
         return self.url_result('http://www.dailymotion.com/video/%s' % info['id'],
                                ie='Dailymotion')

From c1f06d6307109acfa4434a38131c7d9a07d58ab6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:08:33 +0100
Subject: [PATCH 719/974] [macgamestore] Modernize

---
 youtube_dl/extractor/macgamestore.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/macgamestore.py b/youtube_dl/extractor/macgamestore.py
index b818cf50c..3cd4a3a19 100644
--- a/youtube_dl/extractor/macgamestore.py
+++ b/youtube_dl/extractor/macgamestore.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
@@ -13,21 +11,22 @@ class MacGameStoreIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.macgamestore.com/mediaviewer.php?trailer=2450',
-        'file': '2450.m4v',
         'md5': '8649b8ea684b6666b4c5be736ecddc61',
         'info_dict': {
+            'id': '2450',
+            'ext': 'm4v',
             'title': 'Crow',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, video_id, 'Downloading trailer page')
 
-        webpage = self._download_webpage(url, video_id, 'Downloading trailer page')
-
-        if re.search(r'>Missing Media<', webpage) is not None:
-            raise ExtractorError('Trailer %s does not exist' % video_id, expected=True)
+        if '>Missing Media<' in webpage:
+            raise ExtractorError(
+                'Trailer %s does not exist' % video_id, expected=True)
 
         video_title = self._html_search_regex(
             r'<title>MacGameStore: (.*?) Trailer</title>', webpage, 'title')

From 748a0fab8a35fdfde03407d49eb4b41e4be3a1fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:08:50 +0100
Subject: [PATCH 720/974] Remove unused imports

---
 youtube_dl/extractor/defense.py | 3 ---
 youtube_dl/extractor/la7.py     | 2 --
 2 files changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index 1637fa4a7..2b90bf4fc 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -1,8 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
 
 
diff --git a/youtube_dl/extractor/la7.py b/youtube_dl/extractor/la7.py
index 60625c42d..b08f6e3c9 100644
--- a/youtube_dl/extractor/la7.py
+++ b/youtube_dl/extractor/la7.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,

From 9f281cacd2bb1de0dc11dac35685f04dcb5f447c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:13:41 +0100
Subject: [PATCH 721/974] [keezmovies] Fix extraction and modernize test

---
 youtube_dl/extractor/keezmovies.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index 97dcb518a..82eddec51 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -7,10 +7,6 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
-)
-from ..aes import (
-    aes_decrypt_text
 )
 
 
@@ -18,9 +14,10 @@ class KeezMoviesIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/.+?(?P<id>[0-9]+)(?:[/?&]|$)'
     _TEST = {
         'url': 'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
-        'file': '1214711.mp4',
         'md5': '6e297b7e789329923fcf83abb67c9289',
         'info_dict': {
+            'id': '1214711',
+            'ext': 'mp4',
             'title': 'Petite Asian Lady Mai Playing In Bathtub',
             'age_limit': 18,
         }
@@ -39,11 +36,10 @@ class KeezMoviesIE(InfoExtractor):
             embedded_url = mobj.group(1)
             return self.url_result(embedded_url)
 
-        video_title = self._html_search_regex(r'<h1 [^>]*>([^<]+)', webpage, 'title')
-        video_url = compat_urllib_parse.unquote(self._html_search_regex(r'video_url=(.+?)&amp;', webpage, 'video_url'))
-        if 'encrypted=true' in webpage:
-            password = self._html_search_regex(r'video_title=(.+?)&amp;', webpage, 'password')
-            video_url = aes_decrypt_text(video_url, password, 32).decode('utf-8')
+        video_title = self._html_search_regex(
+            r'<h1 [^>]*>([^<]+)', webpage, 'title')
+        video_url = self._html_search_regex(
+            r'(?s)html5VideoPlayer = .*?src="([^"]+)"', webpage, 'video URL')
         path = compat_urllib_parse_urlparse(video_url).path
         extension = os.path.splitext(path)[1][1:]
         format = path.split('/')[4].split('_')[:2]

From 437cac8cc18e541dd1c4c55ef1b19de7d6529ae9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:16:35 +0100
Subject: [PATCH 722/974] [sina] Modernize and simplify

---
 youtube_dl/extractor/sina.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py
index a63d126d4..0891a441f 100644
--- a/youtube_dl/extractor/sina.py
+++ b/youtube_dl/extractor/sina.py
@@ -11,7 +11,7 @@ from ..compat import (
 
 
 class SinaIE(InfoExtractor):
-    _VALID_URL = r'''https?://(.*?\.)?video\.sina\.com\.cn/
+    _VALID_URL = r'''(?x)https?://(.*?\.)?video\.sina\.com\.cn/
                         (
                             (.+?/(((?P<pseudo_id>\d+).html)|(.*?(\#|(vid=)|b/)(?P<id>\d+?)($|&|\-))))
                             |
@@ -23,9 +23,10 @@ class SinaIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://video.sina.com.cn/news/vlist/zt/chczlj2013/?opsubject_id=top12#110028898',
-            'file': '110028898.flv',
             'md5': 'd65dd22ddcf44e38ce2bf58a10c3e71f',
             'info_dict': {
+                'id': '110028898',
+                'ext': 'flv',
                 'title': '《中国新闻》 朝鲜要求巴拿马立即释放被扣船员',
             }
         },
@@ -39,10 +40,6 @@ class SinaIE(InfoExtractor):
         },
     ]
 
-    @classmethod
-    def suitable(cls, url):
-        return re.match(cls._VALID_URL, url, flags=re.VERBOSE) is not None
-
     def _extract_video(self, video_id):
         data = compat_urllib_parse.urlencode({'vid': video_id})
         url_doc = self._download_xml('http://v.iask.com/v_play.php?%s' % data,
@@ -59,7 +56,7 @@ class SinaIE(InfoExtractor):
                 }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
+        mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         if mobj.group('token') is not None:
             # The video id is in the redirected url

From e332772531d15bf15ccfc807085c6a1bdae163f3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:18:52 +0100
Subject: [PATCH 723/974] [servingsys] Modernize

---
 youtube_dl/extractor/servingsys.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/servingsys.py b/youtube_dl/extractor/servingsys.py
index 16dc3736b..c013d678f 100644
--- a/youtube_dl/extractor/servingsys.py
+++ b/youtube_dl/extractor/servingsys.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -13,10 +11,15 @@ class ServingSysIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://bs.serving-sys.com/BurstingPipe/adServer.bs?cn=is&c=23&pl=VAST&pli=5349193&PluID=0&pos=7135&ord=[timestamp]&cim=1?',
+        'info_dict': {
+            'id': '5349193',
+            'title': 'AdAPPter_Hyundai_demo',
+        },
         'playlist': [{
-            'file': '29955898.flv',
             'md5': 'baed851342df6846eb8677a60a011a0f',
             'info_dict': {
+                'id': '29955898',
+                'ext': 'flv',
                 'title': 'AdAPPter_Hyundai_demo (1)',
                 'duration': 74,
                 'tbr': 1378,
@@ -24,9 +27,10 @@ class ServingSysIE(InfoExtractor):
                 'height': 400,
             },
         }, {
-            'file': '29907998.flv',
             'md5': '979b4da2655c4bc2d81aeb915a8c5014',
             'info_dict': {
+                'id': '29907998',
+                'ext': 'flv',
                 'title': 'AdAPPter_Hyundai_demo (2)',
                 'duration': 34,
                 'width': 854,
@@ -37,14 +41,13 @@ class ServingSysIE(InfoExtractor):
         'params': {
             'playlistend': 2,
         },
-        'skip': 'Blocked in the US [sic]',
+        '_skip': 'Blocked in the US [sic]',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        pl_id = mobj.group('id')
-
+        pl_id = self._match_id(url)
         vast_doc = self._download_xml(url, pl_id)
+
         title = vast_doc.find('.//AdTitle').text
         media = vast_doc.find('.//MediaFile').text
         info_url = self._search_regex(r'&adData=([^&]+)&', media, 'info URL')

From c172440ac50b75eaee660a5c00f44c9c8a2f37c0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:21:02 +0100
Subject: [PATCH 724/974] [ringtv] Modernize test definition

---
 youtube_dl/extractor/ringtv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ringtv.py b/youtube_dl/extractor/ringtv.py
index 59dc137cc..efa4afeb6 100644
--- a/youtube_dl/extractor/ringtv.py
+++ b/youtube_dl/extractor/ringtv.py
@@ -6,12 +6,13 @@ from .common import InfoExtractor
 
 
 class RingTVIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
+    _VALID_URL = r'http://(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
     _TEST = {
         "url": "http://ringtv.craveonline.com/news/310833-luis-collazo-says-victor-ortiz-better-not-quit-on-jan-30",
-        "file": "857645.mp4",
         "md5": "d25945f5df41cdca2d2587165ac28720",
         "info_dict": {
+            'id': '857645',
+            'ext': 'mp4',
             "title": 'Video: Luis Collazo says Victor Ortiz "better not quit on Jan. 30" - Ring TV',
             "description": 'Luis Collazo is excited about his Jan. 30 showdown with fellow former welterweight titleholder Victor Ortiz at Barclays Center in his hometown of Brooklyn. The SuperBowl week fight headlines a Golden Boy Live! card on Fox Sports 1.',
         }

From 4e980275b517b101d5cd3646fee6ced5f627da8a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:21:18 +0100
Subject: [PATCH 725/974] [test_download] Remove references to "file" in test
 definitions

---
 test/test_download.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index 412f3dbce..6a149ae4f 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -89,7 +89,7 @@ def generator(test_case):
 
         for tc in test_cases:
             info_dict = tc.get('info_dict', {})
-            if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
+            if not (info_dict.get('id') and info_dict.get('ext')):
                 raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
 
         if 'skip' in test_case:
@@ -116,7 +116,7 @@ def generator(test_case):
         expect_warnings(ydl, test_case.get('expected_warnings', []))
 
         def get_tc_filename(tc):
-            return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
+            return ydl.prepare_filename(tc.get('info_dict', {}))
 
         res_dict = None
 

From a9551e90207a2bb2ef8f35279fc525846e21cbe4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:24:38 +0100
Subject: [PATCH 726/974] [soundcloud:set] Fix test definition

---
 youtube_dl/extractor/soundcloud.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 5d60c4939..c5284fa67 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -246,6 +246,7 @@ class SoundcloudSetIE(SoundcloudIE):
     _TESTS = [{
         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
         'info_dict': {
+            'id': '2284613',
             'title': 'The Royal Concept EP',
         },
         'playlist_mincount': 6,
@@ -279,7 +280,7 @@ class SoundcloudSetIE(SoundcloudIE):
         return {
             '_type': 'playlist',
             'entries': [self._extract_info_dict(track, secret_token=token) for track in info['tracks']],
-            'id': info['id'],
+            'id': '%s' % info['id'],
             'title': info['title'],
         }
 

From 654bd52f5847dc3f1604611f7f1ca3beafee7046 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:25:33 +0100
Subject: [PATCH 727/974] [teletask] Fix test definition

---
 youtube_dl/extractor/teletask.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py
index e54145105..b9e2ef8ca 100644
--- a/youtube_dl/extractor/teletask.py
+++ b/youtube_dl/extractor/teletask.py
@@ -11,6 +11,7 @@ class TeleTaskIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.tele-task.de/archive/video/html5/26168/',
         'info_dict': {
+            'id': '26168',
             'title': 'Duplicate Detection',
         },
         'playlist': [{
@@ -34,7 +35,6 @@ class TeleTaskIE(InfoExtractor):
 
     def _real_extract(self, url):
         lecture_id = self._match_id(url)
-
         webpage = self._download_webpage(url, lecture_id)
 
         title = self._html_search_regex(

From dd8982f19c97a539fc11ee0ee74bb6c6fb6beb57 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:27:16 +0100
Subject: [PATCH 728/974] [wdr] Complete test definition

---
 youtube_dl/extractor/wdr.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 313b9c15d..c90488500 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -71,6 +71,9 @@ class WDRIE(InfoExtractor):
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100.html',
             'playlist_mincount': 146,
+            'info_dict': {
+                'id': 'mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100',
+            }
         }
     ]
 

From acf757f42eea16266a539fa0eb871dae422a1d22 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:33:32 +0100
Subject: [PATCH 729/974] [youtube] Complete test definition

---
 youtube_dl/extractor/youtube.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3b3678c6e..e4b26b84f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1160,6 +1160,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
     }, {
         'url': 'https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
         'info_dict': {
+            'id': 'PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
             'title': 'YDL_Empty_List',
         },
         'playlist_count': 0,
@@ -1168,6 +1169,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
         'info_dict': {
             'title': '29C3: Not my department',
+            'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
         },
         'playlist_count': 95,
     }, {
@@ -1175,6 +1177,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'url': 'PLBB231211A4F62143',
         'info_dict': {
             'title': '[OLD]Team Fortress 2 (Class-based LP)',
+            'id': 'PLBB231211A4F62143',
         },
         'playlist_mincount': 26,
     }, {
@@ -1182,12 +1185,14 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
         'info_dict': {
             'title': 'Uploads from Cauchemar',
+            'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
         },
         'playlist_mincount': 799,
     }, {
         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
         'info_dict': {
             'title': 'YDL_safe_search',
+            'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
         },
         'playlist_count': 2,
     }, {
@@ -1196,6 +1201,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'playlist_count': 4,
         'info_dict': {
             'title': 'JODA15',
+            'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
         }
     }, {
         'note': 'Embedded SWF player',
@@ -1203,12 +1209,14 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'playlist_count': 4,
         'info_dict': {
             'title': 'JODA7',
+            'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
         }
     }, {
         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
         'info_dict': {
-                'title': 'Uploads from Interstellar Movie',
+            'title': 'Uploads from Interstellar Movie',
+            'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
         },
         'playlist_mincout': 21,
     }]
@@ -1314,6 +1322,9 @@ class YoutubeChannelIE(InfoExtractor):
         'note': 'paginated channel',
         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
         'playlist_mincount': 91,
+        'info_dict': {
+            'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+        }
     }]
 
     def extract_videos_from_page(self, page):

From c6df692466aebed1b86c02233b579c03f0888704 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 23:48:23 +0600
Subject: [PATCH 730/974] [rutv] Extract all m3u8 formats

---
 youtube_dl/extractor/rutv.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index a73e6f331..ef766237b 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -162,10 +162,8 @@ class RUTVIE(InfoExtractor):
                         'vbr': int(quality),
                     }
                 elif transport == 'm3u8':
-                    fmt = {
-                        'url': url,
-                        'ext': 'mp4',
-                    }
+                    formats.extend(self._extract_m3u8_formats(url, video_id, 'mp4'))
+                    continue
                 else:
                     fmt = {
                         'url': url

From 027008b14ebe3d41ac7b9940ab73eaed120dab5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 23:49:23 +0600
Subject: [PATCH 731/974] [hls] Fix encode issues on python2 @ Windows

---
 youtube_dl/downloader/hls.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index aa58b52ab..b642786cc 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -11,6 +11,7 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
+    encodeArgument,
     encodeFilename,
 )
 
@@ -21,18 +22,17 @@ class HlsFD(FileDownloader):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        args = [
-            '-y', '-i', url, '-f', 'mp4', '-c', 'copy',
-            '-bsf:a', 'aac_adtstoasc',
-            encodeFilename(tmpfilename, for_subprocess=True)]
-
         ffpp = FFmpegPostProcessor(downloader=self)
         program = ffpp._executable
         if program is None:
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()
-        cmd = [program] + args
+
+        args = [encodeArgument(opt) for opt in ('-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
+        args.append(encodeFilename(tmpfilename, True))
+
+        cmd = [encodeArgument(program)] + args
 
         retval = subprocess.call(cmd)
         if retval == 0:

From 4a3da4ebdb1de5e35b47176822ab9ed3b2762c3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 23:54:38 +0600
Subject: [PATCH 732/974] [hls] Simplify

---
 youtube_dl/downloader/hls.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index b642786cc..e527ee425 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -29,15 +29,15 @@ class HlsFD(FileDownloader):
             return False
         ffpp.check_version()
 
-        args = [encodeArgument(opt) for opt in ('-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
+        args = [
+            encodeArgument(opt)
+            for opt in (program, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
         args.append(encodeFilename(tmpfilename, True))
 
-        cmd = [encodeArgument(program)] + args
-
-        retval = subprocess.call(cmd)
+        retval = subprocess.call(args)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
-            self.to_screen('\r[%s] %s bytes' % (cmd[0], fsize))
+            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
             self.try_rename(tmpfilename, filename)
             self._hook_progress({
                 'downloaded_bytes': fsize,

From 9e3f19919aa3e152760f4cd0a7cad348428f3b35 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 22:38:26 +0100
Subject: [PATCH 733/974] [jsinterp] Beef up and add tests

In preparation for #4822, extend jsinterp by a lot. (We may even have to/want to write a proper interpreter with actual parsing)
---
 test/test_jsinterp.py  |  95 +++++++++++++++++++++++
 youtube_dl/jsinterp.py | 172 ++++++++++++++++++++++++++++++-----------
 2 files changed, 221 insertions(+), 46 deletions(-)
 create mode 100644 test/test_jsinterp.py

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
new file mode 100644
index 000000000..b91b8c492
--- /dev/null
+++ b/test/test_jsinterp.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.jsinterp import JSInterpreter
+
+
+class TestJSInterpreter(unittest.TestCase):
+    def test_basic(self):
+        jsi = JSInterpreter('function x(){;}')
+        self.assertEqual(jsi.call_function('x'), None)
+
+        jsi = JSInterpreter('function x3(){return 42;}')
+        self.assertEqual(jsi.call_function('x3'), 42)
+
+    def test_calc(self):
+        jsi = JSInterpreter('function x4(a){return 2*a+1;}')
+        self.assertEqual(jsi.call_function('x4', 3), 7)
+
+    def test_empty_return(self):
+        jsi = JSInterpreter('function f(){return; y()}')
+        self.assertEqual(jsi.call_function('f'), None)
+
+    def test_morespace(self):
+        jsi = JSInterpreter('function x (a) { return 2 * a + 1 ; }')
+        self.assertEqual(jsi.call_function('x', 3), 7)
+
+        jsi = JSInterpreter('function f () { x =  2  ; return x; }')
+        self.assertEqual(jsi.call_function('f'), 2)
+
+    def test_strange_chars(self):
+        jsi = JSInterpreter('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }')
+        self.assertEqual(jsi.call_function('$_xY1', 20), 21)
+
+    def test_operators(self):
+        jsi = JSInterpreter('function f(){return 1 << 5;}')
+        self.assertEqual(jsi.call_function('f'), 32)
+
+        jsi = JSInterpreter('function f(){return 19 & 21;}')
+        self.assertEqual(jsi.call_function('f'), 17)
+
+        jsi = JSInterpreter('function f(){return 11 >> 2;}')
+        self.assertEqual(jsi.call_function('f'), 2)
+
+    def test_array_access(self):
+        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2] = 7; return x;}')
+        self.assertEqual(jsi.call_function('f'), [5, 2, 7])
+
+    def test_parens(self):
+        jsi = JSInterpreter('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}')
+        self.assertEqual(jsi.call_function('f'), 7)
+
+        jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
+        self.assertEqual(jsi.call_function('f'), 9)
+
+    def test_assignments(self):
+        jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}')
+        self.assertEqual(jsi.call_function('f'), 31)
+
+        jsi = JSInterpreter('function f(){var x = 20; x += 30 + 1; return x;}')
+        self.assertEqual(jsi.call_function('f'), 51)
+
+        jsi = JSInterpreter('function f(){var x = 20; x -= 30 + 1; return x;}')
+        self.assertEqual(jsi.call_function('f'), -11)
+
+    def test_comments(self):
+        jsi = JSInterpreter('''
+        function x() {
+            var x = /* 1 + */ 2;
+            var y = /* 30
+            * 40 */ 50;
+            return x + y;
+        }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 52)
+
+    def test_precedence(self):
+        jsi = JSInterpreter('''
+        function x() {
+            var a = [10, 20, 30, 40, 50];
+            var b = 6;
+            a[0]=a[b%a.length];
+            return a;
+        }''')
+        self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index b4617fbad..49364786b 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1,59 +1,122 @@
 from __future__ import unicode_literals
 
 import json
+import operator
 import re
 
 from .utils import (
     ExtractorError,
 )
 
+_OPERATORS = [
+    ('|', operator.or_),
+    ('^', operator.xor),
+    ('&', operator.and_),
+    ('>>', operator.rshift),
+    ('<<', operator.lshift),
+    ('-', operator.sub),
+    ('+', operator.add),
+    ('%', operator.mod),
+    ('/', operator.div),
+    ('*', operator.mul),
+]
+_ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
+_ASSIGN_OPERATORS.append(('=', lambda cur, right: right))
+
+_NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
+
 
 class JSInterpreter(object):
-    def __init__(self, code):
-        self.code = code
+    def __init__(self, code, objects=None):
+        if objects is None:
+            objects = {}
+        self.code = self._remove_comments(code)
         self._functions = {}
-        self._objects = {}
+        self._objects = objects
 
-    def interpret_statement(self, stmt, local_vars, allow_recursion=20):
+    def _remove_comments(self, code):
+        return re.sub(r'(?s)/\*.*?\*/', '', code)
+
+    def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise ExtractorError('Recursion limit reached')
 
-        if stmt.startswith('var '):
-            stmt = stmt[len('var '):]
-        ass_m = re.match(r'^(?P<out>[a-z]+)(?:\[(?P<index>[^\]]+)\])?' +
-                         r'=(?P<expr>.*)$', stmt)
-        if ass_m:
-            if ass_m.groupdict().get('index'):
-                def assign(val):
-                    lvar = local_vars[ass_m.group('out')]
-                    idx = self.interpret_expression(
-                        ass_m.group('index'), local_vars, allow_recursion)
-                    assert isinstance(idx, int)
-                    lvar[idx] = val
-                    return val
-                expr = ass_m.group('expr')
-            else:
-                def assign(val):
-                    local_vars[ass_m.group('out')] = val
-                    return val
-                expr = ass_m.group('expr')
-        elif stmt.startswith('return '):
-            assign = lambda v: v
-            expr = stmt[len('return '):]
+        should_abort = False
+        stmt = stmt.lstrip()
+        stmt_m = re.match(r'var\s', stmt)
+        if stmt_m:
+            expr = stmt[len(stmt_m.group(0)):]
         else:
-            # Try interpreting it as an expression
-            expr = stmt
-            assign = lambda v: v
+            return_m = re.match(r'return(?:\s+|$)', stmt)
+            if return_m:
+                expr = stmt[len(return_m.group(0)):]
+                should_abort = True
+            else:
+                # Try interpreting it as an expression
+                expr = stmt
 
         v = self.interpret_expression(expr, local_vars, allow_recursion)
-        return assign(v)
+        return v, should_abort
 
     def interpret_expression(self, expr, local_vars, allow_recursion):
+        expr = expr.strip()
+
+        if expr == '':  # Empty expression
+            return None
+
+        if expr.startswith('('):
+            parens_count = 0
+            for m in re.finditer(r'[()]', expr):
+                if m.group(0) == '(':
+                    parens_count += 1
+                else:
+                    parens_count -= 1
+                    if parens_count == 0:
+                        sub_expr = expr[1:m.start()]
+                        sub_result = self.interpret_expression(
+                            sub_expr, local_vars, allow_recursion)
+                        remaining_expr = expr[m.end():].strip()
+                        if not remaining_expr:
+                            return sub_result
+                        else:
+                            expr = json.dumps(sub_result) + remaining_expr
+                        break
+            else:
+                raise ExtractorError('Premature end of parens in %r' % expr)
+
+        for op, opfunc in _ASSIGN_OPERATORS:
+            m = re.match(r'''(?x)
+                (?P<out>%s)(?:\[(?P<index>[^\]]+?)\])?
+                \s*%s
+                (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
+            if not m:
+                continue
+            right_val = self.interpret_expression(
+                m.group('expr'), local_vars, allow_recursion - 1)
+
+            if m.groupdict().get('index'):
+                lvar = local_vars[m.group('out')]
+                idx = self.interpret_expression(
+                    m.group('index'), local_vars, allow_recursion)
+                assert isinstance(idx, int)
+                cur = lvar[idx]
+                val = opfunc(cur, right_val)
+                lvar[idx] = val
+                return val
+            else:
+                cur = local_vars.get(m.group('out'))
+                val = opfunc(cur, right_val)
+                local_vars[m.group('out')] = val
+                return val
+
         if expr.isdigit():
             return int(expr)
 
-        if expr.isalpha():
-            return local_vars[expr]
+        var_m = re.match(
+            r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE,
+            expr)
+        if var_m:
+            return local_vars[var_m.group('name')]
 
         try:
             return json.loads(expr)
@@ -61,7 +124,7 @@ class JSInterpreter(object):
             pass
 
         m = re.match(
-            r'^(?P<var>[$a-zA-Z0-9_]+)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$',
+            r'(?P<var>%s)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
             expr)
         if m:
             variable = m.group('var')
@@ -114,23 +177,31 @@ class JSInterpreter(object):
             return obj[member](argvals)
 
         m = re.match(
-            r'^(?P<in>[a-z]+)\[(?P<idx>.+)\]$', expr)
+            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
         if m:
             val = local_vars[m.group('in')]
             idx = self.interpret_expression(
                 m.group('idx'), local_vars, allow_recursion - 1)
             return val[idx]
 
-        m = re.match(r'^(?P<a>.+?)(?P<op>[%])(?P<b>.+?)$', expr)
-        if m:
-            a = self.interpret_expression(
-                m.group('a'), local_vars, allow_recursion)
-            b = self.interpret_expression(
-                m.group('b'), local_vars, allow_recursion)
-            return a % b
+        for op, opfunc in _OPERATORS:
+            m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
+            if not m:
+                continue
+            x, abort = self.interpret_statement(
+                m.group('x'), local_vars, allow_recursion - 1)
+            if abort:
+                raise ExtractorError(
+                    'Premature left-side return of %s in %r' % (op, expr))
+            y, abort = self.interpret_statement(
+                m.group('y'), local_vars, allow_recursion - 1)
+            if abort:
+                raise ExtractorError(
+                    'Premature right-side return of %s in %r' % (op, expr))
+            return opfunc(x, y)
 
         m = re.match(
-            r'^(?P<func>[a-zA-Z$]+)\((?P<args>[a-z0-9,]+)\)$', expr)
+            r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]+)\)$' % _NAME_RE, expr)
         if m:
             fname = m.group('func')
             argvals = tuple([
@@ -139,6 +210,7 @@ class JSInterpreter(object):
             if fname not in self._functions:
                 self._functions[fname] = self.extract_function(fname)
             return self._functions[fname](argvals)
+
         raise ExtractorError('Unsupported JS expression %r' % expr)
 
     def extract_object(self, objname):
@@ -162,9 +234,11 @@ class JSInterpreter(object):
 
     def extract_function(self, funcname):
         func_m = re.search(
-            (r'(?:function %s|[{;]%s\s*=\s*function)' % (
-                re.escape(funcname), re.escape(funcname))) +
-            r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
+            r'''(?x)
+                (?:function\s+%s|[{;]%s\s*=\s*function)\s*
+                \((?P<args>[^)]*)\)\s*
+                \{(?P<code>[^}]+)\}''' % (
+                re.escape(funcname), re.escape(funcname)),
             self.code)
         if func_m is None:
             raise ExtractorError('Could not find JS function %r' % funcname)
@@ -172,10 +246,16 @@ class JSInterpreter(object):
 
         return self.build_function(argnames, func_m.group('code'))
 
+    def call_function(self, funcname, *args):
+        f = self.extract_function(funcname)
+        return f(args)
+
     def build_function(self, argnames, code):
         def resf(args):
             local_vars = dict(zip(argnames, args))
             for stmt in code.split(';'):
-                res = self.interpret_statement(stmt, local_vars)
+                res, abort = self.interpret_statement(stmt, local_vars)
+                if abort:
+                    break
             return res
         return resf

From 0a7055c90dac10e7f4c3393922bccf546081262b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 22:42:19 +0100
Subject: [PATCH 734/974] release 2015.02.01

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5e6288e8e..e46ca56a9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.30.2'
+__version__ = '2015.02.01'

From 044131ba213be1bf277561432bcb98a07bbdb265 Mon Sep 17 00:00:00 2001
From: Sahebjot singh <sahebjot94@gmail.com>
Date: Mon, 2 Feb 2015 03:49:28 +0530
Subject: [PATCH 735/974] supported sites was not opening

required a .md at the end.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 09da0129e..68b41970a 100644
--- a/README.md
+++ b/README.md
@@ -534,7 +534,7 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 ### How can I detect whether a given URL is supported by youtube-dl?
 
-For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
 It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 

From 8cfb6efe6f4a007a4faab132d08717cea0cdfbaf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 01:49:32 +0100
Subject: [PATCH 736/974] [jsinterp] Correct div command

---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 49364786b..453e2732c 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -17,7 +17,7 @@ _OPERATORS = [
     ('-', operator.sub),
     ('+', operator.add),
     ('%', operator.mod),
-    ('/', operator.div),
+    ('/', operator.truediv),
     ('*', operator.mul),
 ]
 _ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]

From acf2a6e97b53b191edf3f92af1eae300633ea2d9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 01:49:40 +0100
Subject: [PATCH 737/974] release 2015.02.02

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e46ca56a9..1e469d93b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.01'
+__version__ = '2015.02.02'

From 12832049170e8b3bd457706f49e8e09de12a368c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 12:05:39 +0100
Subject: [PATCH 738/974] [http] PEP8 (#4831)

---
 youtube_dl/downloader/http.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 9b023c5c1..49170cf9d 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -102,11 +102,11 @@ class HttpFD(FileDownloader):
                             resume_len = 0
                             open_mode = 'wb'
                             break
-
             except SocketError as e:
                 if e.errno != errno.ECONNRESET:
-                    raise # Not error we are looking for
-                pass
+                    # Connection reset is no problem, just retry
+                    raise
+
             # Retry
             count += 1
             if count <= retries:

From 92bf0bcdf847682cee9f66a5cef4c96c19a100d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 20:52:37 +0600
Subject: [PATCH 739/974] [historicfilms] Add extractor (Closes #4825)

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/historicfilms.py | 46 +++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/historicfilms.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9c14b096f..dabf86ec8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -182,6 +182,7 @@ from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
+from .historicfilms import HistoricFilmsIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hostingbulk import HostingBulkIE
diff --git a/youtube_dl/extractor/historicfilms.py b/youtube_dl/extractor/historicfilms.py
new file mode 100644
index 000000000..40afbe537
--- /dev/null
+++ b/youtube_dl/extractor/historicfilms.py
@@ -0,0 +1,46 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import parse_duration
+
+
+class HistoricFilmsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?historicfilms\.com/(?:tapes/|play)(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.historicfilms.com/tapes/4728',
+        'md5': 'd4a437aec45d8d796a38a215db064e9a',
+        'info_dict': {
+            'id': '4728',
+            'ext': 'mov',
+            'title': 'Historic Films: GP-7',
+            'description': 'md5:1a86a0f3ac54024e419aba97210d959a',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 2096,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        tape_id = self._search_regex(
+            r'class="tapeId">([^<]+)<', webpage, 'tape id')
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._html_search_meta(
+            'thumbnailUrl', webpage, 'thumbnails') or self._og_search_thumbnail(webpage)
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration'))
+
+        video_url = 'http://www.historicfilms.com/video/%s_%s_web.mov' % (tape_id, video_id)
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

From 78271e33197e56452dc60f075bd326cb908e8e7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 21:11:25 +0600
Subject: [PATCH 740/974] [drtv] Extract material id (Closes #4814)

---
 youtube_dl/extractor/drtv.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 8d113a5e8..d5df18d7c 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -25,9 +25,15 @@ class DRTVIE(SubtitlesInfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        programcard = self._download_json(
-            'http://www.dr.dk/mu/programcard/expanded/%s' % video_id, video_id, 'Downloading video JSON')
+        webpage = self._download_webpage(url, video_id)
 
+        video_id = self._search_regex(
+            r'data-(?:material-identifier|episode-slug)="([^"]+)"',
+            webpage, 'video id')
+
+        programcard = self._download_json(
+            'http://www.dr.dk/mu/programcard/expanded/%s' % video_id,
+            video_id, 'Downloading video JSON')
         data = programcard['Data'][0]
 
         title = data['Title']

From 6a66904f8ed469a547d8781fd61b26a97a6ae7aa Mon Sep 17 00:00:00 2001
From: vijayanand nandam <vijay@cybrilla.com>
Date: Mon, 2 Feb 2015 20:51:40 +0530
Subject: [PATCH 741/974] passing source address option to external downloaders

---
 youtube_dl/downloader/external.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index af9fdba75..3d6554fe4 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -45,6 +45,12 @@ class ExternalFD(FileDownloader):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
+    def _source_address(self, command_option):
+        command_part = []
+        if self.ydl.params['source_address'] is not None:
+            command_part = [command_option, self.ydl.params['source_address']]
+        return command_part
+
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
         cmd = self._make_cmd(tmpfilename, info_dict)
@@ -72,6 +78,7 @@ class CurlFD(ExternalFD):
         cmd = [self.exe, '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._source_address('--interface')
         cmd += ['--', info_dict['url']]
         return cmd
 
@@ -81,6 +88,7 @@ class WgetFD(ExternalFD):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._source_address('--bind-address')
         cmd += ['--', info_dict['url']]
         return cmd
 
@@ -96,6 +104,7 @@ class Aria2cFD(ExternalFD):
         cmd += ['--out', os.path.basename(tmpfilename)]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._source_address('--interface')
         cmd += ['--', info_dict['url']]
         return cmd
 

From 469a64cebf534b702e50ad9585e63ae3f8ed5d16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 21:40:52 +0600
Subject: [PATCH 742/974] [downloader/external] Simplify

---
 youtube_dl/downloader/external.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 3d6554fe4..012057ecb 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -47,8 +47,9 @@ class ExternalFD(FileDownloader):
 
     def _source_address(self, command_option):
         command_part = []
-        if self.ydl.params['source_address'] is not None:
-            command_part = [command_option, self.ydl.params['source_address']]
+        source_address = self.ydl.params.get('source_address')
+        if source_address:
+            command_part = [command_option, source_address]
         return command_part
 
     def _call_downloader(self, tmpfilename, info_dict):

From ae7773942ecf816e9592e55bdef7234f41c89c85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 21:51:38 +0600
Subject: [PATCH 743/974] [downloader/external] Simplify

---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 012057ecb..faac11805 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -47,7 +47,7 @@ class ExternalFD(FileDownloader):
 
     def _source_address(self, command_option):
         command_part = []
-        source_address = self.ydl.params.get('source_address')
+        source_address = self.params.get('source_address')
         if source_address:
             command_part = [command_option, source_address]
         return command_part

From 9fffd0469f1353c5f5e96b91a87e34d02ae7491a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:28:18 +0100
Subject: [PATCH 744/974] [options] Mark --fixup as non-experimental and
 correct its help

---
 youtube_dl/downloader/external.py | 2 +-
 youtube_dl/options.py             | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 012057ecb..e47e3bf4c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -48,7 +48,7 @@ class ExternalFD(FileDownloader):
     def _source_address(self, command_option):
         command_part = []
         source_address = self.ydl.params.get('source_address')
-        if source_address:
+        if source_address is None:
             command_part = [command_option, source_address]
         return command_part
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5e4a1ad9b..d5e575964 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -698,10 +698,9 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--fixup',
         metavar='POLICY', dest='fixup', default='detect_or_warn',
-        help='(experimental) Automatically correct known faults of the file. '
+        help='Automatically correct known faults of the file. '
              'One of never (do nothing), warn (only emit a warning), '
-             'detect_or_warn(check whether we can do anything about it, warn '
-             'otherwise')
+             'detect_or_warn(the default; fix file if we can, warn otherwise)')
     postproc.add_option(
         '--prefer-avconv',
         action='store_false', dest='prefer_ffmpeg',

From cc9ca3ba6e0ba24b318ef191fdf84927e773008d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:33:23 +0100
Subject: [PATCH 745/974] [downloader/external] Simplify source_address

'' might actually be passed in, so check for None.
---
 youtube_dl/downloader/external.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index faac11805..ff031d2e0 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -46,11 +46,10 @@ class ExternalFD(FileDownloader):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
     def _source_address(self, command_option):
-        command_part = []
         source_address = self.params.get('source_address')
-        if source_address:
-            command_part = [command_option, source_address]
-        return command_part
+        if source_address is None:
+            return []
+        return [command_option, source_address]
 
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """

From 869b4aeff4239b02ff8f699741902fe7f91b999a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:35:04 +0100
Subject: [PATCH 746/974] release 2015.02.02.1

---
 README.md             | 10 +++++-----
 youtube_dl/version.py |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 68b41970a..d432904fa 100644
--- a/README.md
+++ b/README.md
@@ -368,11 +368,11 @@ which means you can modify it, redistribute it or use it however you like.
     --add-metadata                   write metadata to the video file
     --xattrs                         write metadata to the video file's xattrs
                                      (using dublin core and xdg standards)
-    --fixup POLICY                   (experimental) Automatically correct known
-                                     faults of the file. One of never (do
-                                     nothing), warn (only emit a warning),
-                                     detect_or_warn(check whether we can do
-                                     anything about it, warn otherwise
+    --fixup POLICY                   Automatically correct known faults of the
+                                     file. One of never (do nothing), warn (only
+                                     emit a warning), detect_or_warn(the
+                                     default; fix file if we can, warn
+                                     otherwise)
     --prefer-avconv                  Prefer avconv over ffmpeg for running the
                                      postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1e469d93b..94c181a46 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02'
+__version__ = '2015.02.02.1'

From 3fd45e03bf86c2ad2ef17362452a9a745dc10755 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:43:02 +0100
Subject: [PATCH 747/974] [ntvru] Rename from NTV to clarify the difference
 between n-tv.de and ntv.ru

---
 youtube_dl/extractor/__init__.py          | 2 +-
 youtube_dl/extractor/{ntv.py => ntvru.py} | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)
 rename youtube_dl/extractor/{ntv.py => ntvru.py} (97%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dabf86ec8..bd707e028 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -317,7 +317,7 @@ from .nrk import (
     NRKIE,
     NRKTVIE,
 )
-from .ntv import NTVIE
+from .ntvru import NTVRuIE
 from .nytimes import NYTimesIE
 from .nuvid import NuvidIE
 from .oktoberfesttv import OktoberfestTVIE
diff --git a/youtube_dl/extractor/ntv.py b/youtube_dl/extractor/ntvru.py
similarity index 97%
rename from youtube_dl/extractor/ntv.py
rename to youtube_dl/extractor/ntvru.py
index ee740cd9c..8b9166d48 100644
--- a/youtube_dl/extractor/ntv.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -9,7 +9,7 @@ from ..utils import (
 )
 
 
-class NTVIE(InfoExtractor):
+class NTVRuIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?ntv\.ru/(?P<id>.+)'
 
     _TESTS = [
@@ -92,9 +92,7 @@ class NTVIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         page = self._download_webpage(url, video_id)
 
         video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, page, 'video id')

From 8f4b58d70e29df13d6002579b5c9448acc89d6c1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:48:54 +0100
Subject: [PATCH 748/974] [ntvde] Add new extractor (Fixes #4850)

---
 test/test_utils.py               | 12 ++++++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ntvde.py    | 65 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/ntvru.py    |  1 +
 youtube_dl/utils.py              |  9 ++++-
 5 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/ntvde.py

diff --git a/test/test_utils.py b/test/test_utils.py
index ebec7986f..0ffccd35f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -238,6 +238,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('5 s'), 5)
         self.assertEqual(parse_duration('3 min'), 180)
         self.assertEqual(parse_duration('2.5 hours'), 9000)
+        self.assertEqual(parse_duration('02:03:04'), 7384)
+        self.assertEqual(parse_duration('01:02:03:04'), 93784)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
@@ -371,6 +373,16 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{"abc": true}')
         self.assertEqual(json.loads(on), {'abc': True})
 
+        # Ignore JavaScript code as well
+        on = js_to_json('''{
+            "x": 1,
+            y: "a",
+            z: some.code
+        }''')
+        d = json.loads(on)
+        self.assertEqual(d['x'], 1)
+        self.assertEqual(d['y'], 'a')
+
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
         self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd707e028..205bd4338 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -317,6 +317,7 @@ from .nrk import (
     NRKIE,
     NRKTVIE,
 )
+from .ntvde import NTVDeIE
 from .ntvru import NTVRuIE
 from .nytimes import NYTimesIE
 from .nuvid import NuvidIE
diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py
new file mode 100644
index 000000000..a4f628fcd
--- /dev/null
+++ b/youtube_dl/extractor/ntvde.py
@@ -0,0 +1,65 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    parse_duration,
+)
+
+
+class NTVDeIE(InfoExtractor):
+    IE_NAME = 'n-tv.de'
+    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/videos/[^/?#]+/[^/?#]+-article(?P<id>.+)\.html'
+
+    _TESTS = [{
+        'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
+        'md5': 'd37b7df1eea32265c51a062499ca488f',
+        'info_dict': {
+            'id': '14438086',
+            'ext': 'mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'title': 'Schnee und Glätte führen zu zahlreichen Unfällen und Staus',
+            'alt_title': 'Winterchaos auf deutschen Straßen',
+            'description': 'Schnee und Glätte sorgen deutschlandweit für einen chaotischen Start in die Woche: Auf den Straßen kommt es zu kilometerlangen Staus und Dutzenden Glätteunfällen. In Düsseldorf und München wirbelt der Schnee zudem den Flugplan durcheinander. Dutzende Flüge landen zu spät, einige fallen ganz aus.',
+            'duration': 4020,
+            'timestamp': 1422892797,
+            'upload_date': '20150202',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        info = self._parse_json(self._search_regex(
+            r'(?s)ntv.pageInfo.article =\s(\{.*?\});', webpage, 'info'),
+            video_id, transform_source=js_to_json)
+        timestamp = int_or_none(info.get('publishedDateAsUnixTimeStamp'))
+        vdata = self._parse_json(self._search_regex(
+            r'(?s)\$\(\s*"\#player"\s*\)\s*\.data\(\s*"player",\s*(\{.*?\})\);',
+            webpage, 'player data'),
+            video_id, transform_source=js_to_json)
+        duration = parse_duration(vdata.get('duration'))
+        formats = [{
+            'format_id': 'flash',
+            'url': 'rtmp://fms.n-tv.de/' + vdata['video'],
+        }, {
+            'format_id': 'mp4',
+            'url': 'http://video.n-tv.de' + vdata['videoMp4'],
+        }]
+        m3u8_url = 'http://video.n-tv.de' + vdata['videoM3u8']
+        formats.extend(self._extract_m3u8_formats(m3u8_url, video_id))
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['headline'],
+            'description': info.get('intro'),
+            'alt_title': info.get('kicker'),
+            'timestamp': timestamp,
+            'thumbnail': vdata.get('html5VideoPoster'),
+            'duration': duration,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 8b9166d48..963bb715a 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -10,6 +10,7 @@ from ..utils import (
 
 
 class NTVRuIE(InfoExtractor):
+    IE_NAME = 'ntv.ru'
     _VALID_URL = r'http://(?:www\.)?ntv\.ru/(?P<id>.+)'
 
     _TESTS = [
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4ade0554e..251074bf5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1275,7 +1275,10 @@ def parse_duration(s):
             (?P<only_hours>[0-9.]+)\s*(?:hours?)|
 
             (?:
-                (?:(?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*)?
+                (?:
+                    (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
+                    (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
+                )?
                 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
             )?
             (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
@@ -1293,6 +1296,8 @@ def parse_duration(s):
         res += int(m.group('mins')) * 60
     if m.group('hours'):
         res += int(m.group('hours')) * 60 * 60
+    if m.group('days'):
+        res += int(m.group('days')) * 24 * 60 * 60
     if m.group('ms'):
         res += float(m.group('ms'))
     return res
@@ -1543,7 +1548,7 @@ def js_to_json(code):
     res = re.sub(r'''(?x)
         "(?:[^"\\]*(?:\\\\|\\")?)*"|
         '(?:[^'\\]*(?:\\\\|\\')?)*'|
-        [a-zA-Z_][a-zA-Z_0-9]*
+        [a-zA-Z_][.a-zA-Z_0-9]*
         ''', fix_kv, code)
     res = re.sub(r',(\s*\])', lambda m: m.group(1), res)
     return res

From 7a7bd19c4537a4abd0ccea166610c9f2353046a4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:57:48 +0100
Subject: [PATCH 749/974] [n-tv.de] Use native m3u8 as best format

---
 youtube_dl/extractor/ntvde.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py
index a4f628fcd..d2cfe0961 100644
--- a/youtube_dl/extractor/ntvde.py
+++ b/youtube_dl/extractor/ntvde.py
@@ -15,7 +15,7 @@ class NTVDeIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
-        'md5': 'd37b7df1eea32265c51a062499ca488f',
+        'md5': '6ef2514d4b1e8e03ca24b49e2f167153',
         'info_dict': {
             'id': '14438086',
             'ext': 'mp4',
@@ -46,11 +46,14 @@ class NTVDeIE(InfoExtractor):
             'format_id': 'flash',
             'url': 'rtmp://fms.n-tv.de/' + vdata['video'],
         }, {
-            'format_id': 'mp4',
+            'format_id': 'mobile',
             'url': 'http://video.n-tv.de' + vdata['videoMp4'],
+            'tbr': 400,  # estimation
         }]
         m3u8_url = 'http://video.n-tv.de' + vdata['videoM3u8']
-        formats.extend(self._extract_m3u8_formats(m3u8_url, video_id))
+        formats.extend(self._extract_m3u8_formats(
+            m3u8_url, video_id, ext='mp4',
+            entry_protocol='m3u8_native', preference=0))
         self._sort_formats(formats)
 
         return {

From f920ce295e65dc1446120c9375ef11bfd6cede26 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:58:17 +0100
Subject: [PATCH 750/974] [ntvru] Remove unused import

---
 youtube_dl/extractor/ntvru.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 963bb715a..0ab8d5100 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     unescapeHTML

From c21b1fbeeb0305fbf7804fcad9d4aa2dc6288de6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:58:58 +0100
Subject: [PATCH 751/974] release 2015.02.02.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 94c181a46..c1d17844e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.1'
+__version__ = '2015.02.02.2'

From 7d3d06a16cb585be35c2152f4bd4cba9c32081a3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 22:48:12 +0100
Subject: [PATCH 752/974] [vevo] Restore SMIL support (#3656)

---
 youtube_dl/extractor/vevo.py | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 43f6b029d..c17094f81 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -9,6 +9,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    int_or_none,
 )
 
 
@@ -192,9 +193,29 @@ class VevoIE(InfoExtractor):
         # Download via HLS API
         formats.extend(self._download_api_formats(video_id))
 
+        # Download SMIL
+        smil_blocks = sorted((
+            f for f in video_info['videoVersions']
+            if f['sourceType'] == 13),
+            key=lambda f: f['version'])
+        smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
+            self._SMIL_BASE_URL, video_id, video_id.lower())
+        if smil_blocks:
+            smil_url_m = self._search_regex(
+                r'url="([^"]+)"', smil_blocks[-1]['data'], 'SMIL URL',
+                default=None)
+            if smil_url_m is not None:
+                smil_url = smil_url_m
+        if smil_url:
+            smil_xml = self._download_webpage(
+                smil_url, video_id, 'Downloading SMIL info', fatal=False)
+            if smil_xml:
+                formats.extend(self._formats_from_smil(smil_xml))
+
         self._sort_formats(formats)
-        timestamp_ms = int(self._search_regex(
-            r'/Date\((\d+)\)/', video_info['launchDate'], 'launch date'))
+        timestamp_ms = int_or_none(self._search_regex(
+            r'/Date\((\d+)\)/',
+            video_info['launchDate'], 'launch date', fatal=False))
 
         return {
             'id': video_id,

From 3a4cca687f674fd5b93fafbe80453b23ef7ea748 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 22:56:15 +0100
Subject: [PATCH 753/974] release 2015.02.02.3

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c1d17844e..dadec36d3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.2'
+__version__ = '2015.02.02.3'

From 531572578e542fcee26f1a30fd9f87ad158e6594 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:04:39 +0100
Subject: [PATCH 754/974] [normalboots] Modernize

---
 youtube_dl/extractor/normalboots.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index c13ff0d65..78e75266f 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class NormalbootsIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?normalboots\.com/video/(?P<videoid>[0-9a-z-]*)/?$'
+    _VALID_URL = r'http://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$'
     _TEST = {
         'url': 'http://normalboots.com/video/home-alone-games-jontron/',
         'md5': '8bf6de238915dd501105b44ef5f1e0f6',
@@ -30,19 +30,22 @@ class NormalbootsIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        video_uploader = self._html_search_regex(r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>',
-                                                 webpage, 'uploader')
-        raw_upload_date = self._html_search_regex('<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
-                                                  webpage, 'date')
-        video_upload_date = unified_strdate(raw_upload_date)
 
-        player_url = self._html_search_regex(r'<iframe\swidth="[0-9]+"\sheight="[0-9]+"\ssrc="(?P<url>[\S]+)"', webpage, 'url')
+        video_uploader = self._html_search_regex(
+            r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>',
+            webpage, 'uploader', fatal=False)
+        video_upload_date = unified_strdate(self._html_search_regex(
+            r'<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
+            webpage, 'date', fatal=False))
+
+        player_url = self._html_search_regex(
+            r'<iframe\swidth="[0-9]+"\sheight="[0-9]+"\ssrc="(?P<url>[\S]+)"',
+            webpage, 'player url')
         player_page = self._download_webpage(player_url, video_id)
-        video_url = self._html_search_regex(r"file:\s'(?P<file>[^']+\.mp4)'", player_page, 'file')
+        video_url = self._html_search_regex(
+            r"file:\s'(?P<file>[^']+\.mp4)'", player_page, 'file')
 
         return {
             'id': video_id,

From a57b562cff57a5cbd06cb22b37962861a423e428 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 2 Feb 2015 23:13:28 +0100
Subject: [PATCH 755/974] [nfl] Add support for articles pages (fixes #4848)

---
 youtube_dl/extractor/nfl.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 606e2294e..2684dd250 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -46,7 +46,18 @@ class NFLIE(InfoExtractor):
                 'timestamp': 1388354455,
                 'thumbnail': 're:^https?://.*\.jpg$',
             }
-        }
+        },
+        {
+            'url': 'http://www.nfl.com/news/story/0ap3000000467586/article/patriots-seahawks-involved-in-lategame-skirmish',
+            'info_dict': {
+                'id': '0ap3000000467607',
+                'ext': 'mp4',
+                'title': 'Frustrations flare on the field',
+                'description': 'Emotions ran high at the end of the Super Bowl on both sides of the ball after a dramatic finish.',
+                'timestamp': 1422850320,
+                'upload_date': '20150202',
+            },
+        },
     ]
 
     @staticmethod
@@ -80,7 +91,11 @@ class NFLIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         config_url = NFLIE.prepend_host(host, self._search_regex(
-            r'(?:config|configURL)\s*:\s*"([^"]+)"', webpage, 'config URL'))
+            r'(?:config|configURL)\s*:\s*"([^"]+)"', webpage, 'config URL',
+            default='static/content/static/config/video/config.json'))
+        # For articles, the id in the url is not the video id
+        video_id = self._search_regex(
+            r'contentId\s*:\s*"([^"]+)"', webpage, 'video id', default=video_id)
         config = self._download_json(config_url, video_id,
                                      note='Downloading player config')
         url_template = NFLIE.prepend_host(

From 995029a142cd0046ee43b583f2d09fc3d5fa321a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:38:35 +0100
Subject: [PATCH 756/974] [nerdist] Add new extractor (Fixes #4851)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/common.py   |  7 ++-
 youtube_dl/extractor/nerdist.py  | 80 ++++++++++++++++++++++++++++++++
 3 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/nerdist.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 205bd4338..5dcb14feb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -285,6 +285,7 @@ from .ndr import NDRIE
 from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
+from .nerdist import NerdistIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
 from .nextmedia import (
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8b4ef3f09..9e517e3ac 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -860,10 +860,13 @@ class InfoExtractor(object):
         return formats
 
     # TODO: improve extraction
-    def _extract_smil_formats(self, smil_url, video_id):
+    def _extract_smil_formats(self, smil_url, video_id, fatal=True):
         smil = self._download_xml(
             smil_url, video_id, 'Downloading SMIL file',
-            'Unable to download SMIL file')
+            'Unable to download SMIL file', fatal=fatal)
+        if smil is False:
+            assert not fatal
+            return []
 
         base = smil.find('./head/meta').get('base')
 
diff --git a/youtube_dl/extractor/nerdist.py b/youtube_dl/extractor/nerdist.py
new file mode 100644
index 000000000..c6dc34be4
--- /dev/null
+++ b/youtube_dl/extractor/nerdist.py
@@ -0,0 +1,80 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    determine_ext,
+    parse_iso8601,
+    xpath_text,
+)
+
+
+class NerdistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nerdist\.com/vepisode/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://www.nerdist.com/vepisode/exclusive-which-dc-characters-w',
+        'md5': '3698ed582931b90d9e81e02e26e89f23',
+        'info_dict': {
+            'display_id': 'exclusive-which-dc-characters-w',
+            'id': 'RPHpvJyr',
+            'ext': 'mp4',
+            'title': 'Your TEEN TITANS Revealed! Who\'s on the show?',
+            'thumbnail': 're:^https?://.*/thumbs/.*\.jpg$',
+            'description': 'Exclusive: Find out which DC Comics superheroes will star in TEEN TITANS Live-Action TV Show on Nerdist News with Jessica Chobot!',
+            'uploader': 'Eric Diaz',
+            'upload_date': '20150202',
+            'timestamp': 1422892808,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            r'''(?x)<script\s+(?:type="text/javascript"\s+)?
+                src="https?://content\.nerdist\.com/players/([a-zA-Z0-9_]+)-''',
+            webpage, 'video ID')
+        timestamp = parse_iso8601(self._html_search_meta(
+            'shareaholic:article_published_time', webpage, 'upload date'))
+        uploader = self._html_search_meta(
+            'shareaholic:article_author_name', webpage, 'article author')
+
+        doc = self._download_xml(
+            'http://content.nerdist.com/jw6/%s.xml' % video_id, video_id)
+        video_info = doc.find('.//item')
+        title = xpath_text(video_info, './title', fatal=True)
+        description = xpath_text(video_info, './description')
+        thumbnail = xpath_text(
+            video_info, './{http://rss.jwpcdn.com/}image', 'thumbnail')
+
+        formats = []
+        for source in video_info.findall('./{http://rss.jwpcdn.com/}source'):
+            vurl = source.attrib['file']
+            ext = determine_ext(vurl)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    vurl, video_id, entry_protocol='m3u8_native', ext='mp4',
+                    preference=0))
+            elif ext == 'smil':
+                formats.extend(self._extract_smil_formats(
+                    vurl, video_id, fatal=False
+                ))
+            else:
+                formats.append({
+                    'format_id': ext,
+                    'url': vurl,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'formats': formats,
+            'uploader': uploader,
+        }

From 5c4862bad4ba38f4ddd86d4fb1305aa5b82b2bd6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:38:45 +0100
Subject: [PATCH 757/974] [normalboots] Remove unused import

---
 youtube_dl/extractor/normalboots.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index 78e75266f..5952d136f 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 from ..utils import (

From 55898ad2cf4a5cf7628cf921fb5785ed0c7dceff Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:39:03 +0100
Subject: [PATCH 758/974] release 2015.02.02.4

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dadec36d3..22464a358 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.3'
+__version__ = '2015.02.02.4'

From c1147c05e172f8d8c2b2b8d752eda3f385f1bed1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:47:14 +0100
Subject: [PATCH 759/974] [brightcove] Fix up more generically invalid XML
 (Fixes #4849)

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 003152c4e..ea0969d4d 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -108,7 +108,7 @@ class BrightcoveIE(InfoExtractor):
         """
 
         # Fix up some stupid HTML, see https://github.com/rg3/youtube-dl/issues/1553
-        object_str = re.sub(r'(<param name="[^"]+" value="[^"]+")>',
+        object_str = re.sub(r'(<param(?:\s+[a-zA-Z0-9_]+="[^"]*")*)>',
                             lambda m: m.group(1) + '/>', object_str)
         # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
         object_str = object_str.replace('<--', '<!--')

From d0eca21021b6321197caccab110de13843be6e04 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:47:19 +0100
Subject: [PATCH 760/974] release 2015.02.02.5

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 22464a358..46af42171 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.4'
+__version__ = '2015.02.02.5'

From d200b11c7ea34138d9b8a778fd2dd9226308b48f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:14:42 +0100
Subject: [PATCH 761/974] [Makefile] Simplify clean/cleanall

---
 Makefile              | 5 +----
 devscripts/release.sh | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 578079879..0636fc4cb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json CONTRIBUTING.md.tmp
-
-cleanall: clean
-	rm -f youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin
diff --git a/devscripts/release.sh b/devscripts/release.sh
index 691517ceb..47f54c414 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -35,7 +35,7 @@ if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $us
 if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
 
 /bin/echo -e "\n### First of all, testing..."
-make cleanall
+make clean
 if $skip_tests ; then
     echo 'SKIPPING TESTS'
 else

From 2cf0ecac7b11e68a94990ce7c0e4a661cc2d8c74 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:16:45 +0100
Subject: [PATCH 762/974] [ffmpeg] --add-metadata: Set comment and purl fields
 (Fixes #4847)

---
 youtube_dl/postprocessor/ffmpeg.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 855d1e6db..4a4422c5a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -511,8 +511,9 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             metadata['artist'] = info['uploader_id']
         if info.get('description') is not None:
             metadata['description'] = info['description']
+            metadata['comment'] = info['description']
         if info.get('webpage_url') is not None:
-            metadata['comment'] = info['webpage_url']
+            metadata['purl'] = info['webpage_url']
 
         if not metadata:
             self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')

From 330537d08aa9875c9d66354f2c9bed7aeaf86297 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:20:57 +0100
Subject: [PATCH 763/974] [README] typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d432904fa..c1221417e 100644
--- a/README.md
+++ b/README.md
@@ -728,7 +728,7 @@ In particular, every site support request issue should only pertain to services
 
 ###  Is anyone going to need the feature?
 
-Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
+Only post features that you (or an incapacitated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
 
 ###  Is your question about youtube-dl?
 

From ee623d9247752e11e98829834ba0e3308a21df52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:22:17 +0100
Subject: [PATCH 764/974] [descripts/release] Regenerate auxiliary
 documentation on build as well

---
 devscripts/release.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index 47f54c414..61806961c 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -45,9 +45,9 @@ fi
 /bin/echo -e "\n### Changing version in version.py..."
 sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
 
-/bin/echo -e "\n### Committing README.md and youtube_dl/version.py..."
-make README.md
-git add README.md youtube_dl/version.py
+/bin/echo -e "\n### Committing documentation and youtube_dl/version.py..."
+make README.md CONTRIBUTING.md supportedsites
+git add README.md CONTRIBUTING.md docs/supportedsites.md youtube_dl/version.py
 git commit -m "release $version"
 
 /bin/echo -e "\n### Now tagging, signing and pushing..."

From 4c1a017e6961c18fdef13d49bcdd0daa08ab0575 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:22:52 +0100
Subject: [PATCH 765/974] release 2015.02.03

---
 CONTRIBUTING.md        |  6 +++--
 docs/supportedsites.md | 59 +++++++++++++++++++++++++++++++++++++-----
 youtube_dl/version.py  |  2 +-
 3 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7917abfc6..351229f21 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,4 +1,6 @@
-Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+**Please include the full output of youtube-dl when run with `-v`**.
+
+The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
 Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 
@@ -122,7 +124,7 @@ If you want to add support for a new site, you can follow this quick list (assum
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
-8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
+8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index dbbf4a797..0cebe3981 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -9,6 +9,7 @@
  - **8tracks**
  - **9gag**
  - **abc.net.au**
+ - **Abc7News**
  - **AcademicEarth:Course**
  - **AddAnime**
  - **AdobeTV**
@@ -16,9 +17,12 @@
  - **Aftonbladet**
  - **AlJazeera**
  - **Allocine**
+ - **AlphaPorno**
  - **anitube.se**
  - **AnySex**
  - **Aparat**
+ - **AppleDailyAnimationNews**
+ - **AppleDailyRealtimeNews**
  - **AppleTrailers**
  - **archive.org**: archive.org videos
  - **ARD**
@@ -30,8 +34,10 @@
  - **arte.tv:ddc**
  - **arte.tv:embed**
  - **arte.tv:future**
+ - **AtresPlayer**
+ - **ATTTechChannel**
  - **audiomack**
- - **AUEngine**
+ - **audiomack:album**
  - **Azubu**
  - **bambuser**
  - **bambuser:channel**
@@ -71,8 +77,10 @@
  - **cmt.com**
  - **CNET**
  - **CNN**
+ - **CNNArticle**
  - **CNNBlogs**
  - **CollegeHumor**
+ - **CollegeRama**
  - **ComCarCoff**
  - **ComedyCentral**
  - **ComedyCentralShows**: The Daily Show / The Colbert Report
@@ -82,23 +90,27 @@
  - **Crunchyroll**
  - **crunchyroll:playlist**
  - **CSpan**: C-SPAN
+ - **CtsNews**
  - **culturebox.francetvinfo.fr**
  - **dailymotion**
  - **dailymotion:playlist**
  - **dailymotion:user**
  - **daum.net**
  - **DBTV**
+ - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
  - **Discovery**
  - **divxstage**: DivxStage
  - **Dotsub**
+ - **DRBonanza**
  - **Dropbox**
  - **DrTuber**
  - **DRTV**
  - **Dump**
  - **dvtv**: http://video.aktualne.cz/
  - **EbaumsWorld**
+ - **EchoMsk**
  - **eHow**
  - **Einthusan**
  - **eitb.tv**
@@ -108,6 +120,7 @@
  - **EMPFlix**
  - **Engadget**
  - **Eporner**
+ - **EroProfile**
  - **Escapist**
  - **EveryonesMixtape**
  - **exfm**: ex.fm
@@ -143,6 +156,7 @@
  - **GDCVault**
  - **generic**: Generic downloader that works on some sites
  - **GiantBomb**
+ - **Giga**
  - **Glide**: Glide mobile video messages (glide.me)
  - **Globo**
  - **GodTube**
@@ -153,9 +167,14 @@
  - **Grooveshark**
  - **Groupon**
  - **Hark**
+ - **HearThisAt**
  - **Heise**
+ - **HellPorno**
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
+ - **HistoricFilms**
+ - **hitbox**
+ - **hitbox:live**
  - **HornBunny**
  - **HostingBulk**
  - **HotNewHipHop**
@@ -182,6 +201,7 @@
  - **jpopsuki.tv**
  - **Jukebox**
  - **Kankan**
+ - **Karaoketv**
  - **keek**
  - **KeezMovies**
  - **KhanAcademy**
@@ -195,6 +215,7 @@
  - **LiveLeak**
  - **livestream**
  - **livestream:original**
+ - **LnkGo**
  - **lrt.lt**
  - **lynda**: lynda.com videos
  - **lynda:course**: lynda.com online courses
@@ -235,6 +256,7 @@
  - **MySpass**
  - **myvideo**
  - **MyVidster**
+ - **n-tv.de**
  - **Naver**
  - **NBA**
  - **NBC**
@@ -242,11 +264,16 @@
  - **ndr**: NDR.de - Mediathek
  - **NDTV**
  - **NerdCubedFeed**
+ - **Nerdist**
+ - **Netzkino**
  - **Newgrounds**
  - **Newstube**
+ - **NextMedia**
+ - **NextMediaActionNews**
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **nhl.com**
+ - **nhl.com:news**: NHL news
  - **nhl.com:videocenter**: NHL videocenter category
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
@@ -257,18 +284,20 @@
  - **Nowness**
  - **nowvideo**: NowVideo
  - **npo.nl**
+ - **npo.nl:live**
  - **NRK**
  - **NRKTV**
- - **NTV**
+ - **ntv.ru**
  - **Nuvid**
  - **NYTimes**
  - **ocw.mit.edu**
  - **OktoberfestTV**
  - **on.aol.com**
  - **Ooyala**
+ - **OpenFilm**
+ - **orf:fm4**: radio FM4
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
- - **ORFFM4**: radio FM4
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **PBS**
@@ -290,6 +319,7 @@
  - **Pyvideo**
  - **QuickVid**
  - **radio.de**
+ - **radiobremen**
  - **radiofrance**
  - **Rai**
  - **RBMARadio**
@@ -300,6 +330,8 @@
  - **RottenTomatoes**
  - **Roxwel**
  - **RTBF**
+ - **Rte**
+ - **RTL2**
  - **RTLnow**
  - **rtlxl.nl**
  - **RTP**
@@ -309,6 +341,7 @@
  - **RUHD**
  - **rutube**: Rutube videos
  - **rutube:channel**: Rutube channels
+ - **rutube:embed**: Rutube embedded videos
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
@@ -351,11 +384,12 @@
  - **Sport5**
  - **SportBox**
  - **SportDeutschland**
- - **SRMediathek**: Süddeutscher Rundfunk
+ - **SRMediathek**: Saarländischer Rundfunk
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
  - **streamcloud.eu**
  - **StreamCZ**
+ - **StreetVoice**
  - **SunPorno**
  - **SWRMediathek**
  - **Syfy**
@@ -375,7 +409,9 @@
  - **TeleBruxelles**
  - **telecinco.es**
  - **TeleMB**
+ - **TeleTask**
  - **TenPlay**
+ - **TestTube**
  - **TF1**
  - **TheOnion**
  - **ThePlatform**
@@ -403,8 +439,15 @@
  - **tv.dfb.de**
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
+ - **tvp.pl:Series**
  - **TVPlay**: TV3Play and related services
- - **Twitch**
+ - **twitch:bookmarks**
+ - **twitch:chapter**
+ - **twitch:past_broadcasts**
+ - **twitch:profile**
+ - **twitch:stream**
+ - **twitch:video**
+ - **twitch:vod**
  - **Ubu**
  - **udemy**
  - **udemy:course**
@@ -433,6 +476,8 @@
  - **videoweed**: VideoWeed
  - **Vidme**
  - **Vidzi**
+ - **vier**
+ - **vier:videos**
  - **viki**
  - **vimeo**
  - **vimeo:album**
@@ -460,6 +505,7 @@
  - **WDR**
  - **wdr:mobile**
  - **WDRMaus**: Sendung mit der Maus
+ - **WebOfStories**
  - **Weibo**
  - **Wimp**
  - **Wistia**
@@ -472,7 +518,9 @@
  - **XNXX**
  - **XTube**
  - **XTubeUser**: XTube user profile
+ - **Xuite**
  - **XVideos**
+ - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
  - **YesJapan**
  - **Ynet**
@@ -491,7 +539,6 @@
  - **youtube:search_url**: YouTube.com search URLs
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
- - **youtube:toplist**: YouTube.com top lists, "yttoplist:{channel}:{list title}" (Example: "yttoplist:music:Top Tracks")
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
  - **youtube:watch_later**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
  - **ZDF**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 46af42171..a96e74629 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.5'
+__version__ = '2015.02.03'

From 23d9ded655f45c5a21c8d93ad167a072322ae88f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:15:05 +0100
Subject: [PATCH 766/974] [franceculture] Rewrite for new HTML scheme (Fixes
 #4853)

---
 test/helper.py                        | 10 ++++
 youtube_dl/extractor/franceculture.py | 72 ++++++++++++---------------
 2 files changed, 42 insertions(+), 40 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 805467c82..651ef99b9 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -103,6 +103,16 @@ def expect_info_dict(self, got_dict, expected_dict):
             self.assertTrue(
                 match_rex.match(got),
                 'field %s (value: %r) should match %r' % (info_field, got, match_str))
+        elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
+            got = got_dict.get(info_field)
+            start_str = expected[len('startswith:'):]
+            self.assertTrue(
+                isinstance(got, compat_str),
+                'Expected a %s object, but got %s for field %s' % (
+                    compat_str.__name__, type(got).__name__, info_field))
+            self.assertTrue(
+                got.startswith(start_str),
+                'field %s (value: %r) should start with %r' % (info_field, got, start_str))
         elif isinstance(expected, type):
             got = got_dict.get(info_field)
             self.assertTrue(isinstance(got, expected),
diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index 0c2972162..1e83a4e7e 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -1,77 +1,69 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..compat import (
-    compat_parse_qs,
     compat_urlparse,
 )
+from ..utils import (
+    determine_ext,
+    int_or_none,
+)
 
 
 class FranceCultureIE(InfoExtractor):
-    _VALID_URL = r'(?P<baseurl>http://(?:www\.)?franceculture\.fr/)player/reecouter\?play=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?franceculture\.fr/player/reecouter\?play=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.franceculture.fr/player/reecouter?play=4795174',
         'info_dict': {
             'id': '4795174',
             'ext': 'mp3',
             'title': 'Rendez-vous au pays des geeks',
+            'alt_title': 'Carnet nomade | 13-14',
             'vcodec': 'none',
-            'uploader': 'Colette Fellous',
             'upload_date': '20140301',
-            'duration': 3601,
             'thumbnail': r're:^http://www\.franceculture\.fr/.*/images/player/Carnet-nomade\.jpg$',
-            'description': 'Avec :Jean-Baptiste Péretié pour son documentaire sur Arte "La revanche des « geeks », une enquête menée aux Etats-Unis dans la S ...',
+            'description': 'startswith:Avec :Jean-Baptiste Péretié pour son documentaire sur Arte "La revanche des « geeks », une enquête menée aux Etats',
+            'timestamp': 1393700400,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        baseurl = mobj.group('baseurl')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        params_code = self._search_regex(
-            r"<param name='movie' value='/sites/all/modules/rf/rf_player/swf/loader.swf\?([^']+)' />",
-            webpage, 'parameter code')
-        params = compat_parse_qs(params_code)
-        video_url = compat_urlparse.urljoin(baseurl, params['urlAOD'][0])
+
+        video_path = self._search_regex(
+            r'<a id="player".*?href="([^"]+)"', webpage, 'video path')
+        video_url = compat_urlparse.urljoin(url, video_path)
+        timestamp = int_or_none(self._search_regex(
+            r'<a id="player".*?data-date="([0-9]+)"',
+            webpage, 'upload date', fatal=False))
+        thumbnail = self._search_regex(
+            r'<a id="player".*?>\s+<img src="([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
 
         title = self._html_search_regex(
-            r'<h1 class="title[^"]+">(.+?)</h1>', webpage, 'title')
+            r'<span class="title-diffusion">(.*?)</span>', webpage, 'title')
+        alt_title = self._html_search_regex(
+            r'<span class="title">(.*?)</span>',
+            webpage, 'alt_title', fatal=False)
+        description = self._html_search_regex(
+            r'<span class="description">(.*?)</span>',
+            webpage, 'description', fatal=False)
+
         uploader = self._html_search_regex(
             r'(?s)<div id="emission".*?<span class="author">(.*?)</span>',
-            webpage, 'uploader', fatal=False)
-        thumbnail_part = self._html_search_regex(
-            r'(?s)<div id="emission".*?<img src="([^"]+)"', webpage,
-            'thumbnail', fatal=False)
-        if thumbnail_part is None:
-            thumbnail = None
-        else:
-            thumbnail = compat_urlparse.urljoin(baseurl, thumbnail_part)
-        description = self._html_search_regex(
-            r'(?s)<p class="desc">(.*?)</p>', webpage, 'description')
-
-        info = json.loads(params['infoData'][0])[0]
-        duration = info.get('media_length')
-        upload_date_candidate = info.get('media_section5')
-        upload_date = (
-            upload_date_candidate
-            if (upload_date_candidate is not None and
-                re.match(r'[0-9]{8}$', upload_date_candidate))
-            else None)
+            webpage, 'uploader', default=None)
+        vcodec = 'none' if determine_ext(video_url.lower()) == 'mp3' else None
 
         return {
             'id': video_id,
             'url': video_url,
-            'vcodec': 'none' if video_url.lower().endswith('.mp3') else None,
-            'duration': duration,
+            'vcodec': vcodec,
             'uploader': uploader,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'title': title,
+            'alt_title': alt_title,
             'thumbnail': thumbnail,
             'description': description,
         }

From d86204230157fc7a8a6800b7b814e8e3bbfbdf20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:18:32 +0100
Subject: [PATCH 767/974] [aftonbladet] Modernize

---
 youtube_dl/extractor/aftonbladet.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index cfc7370ae..8442019ea 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -21,9 +19,7 @@ class AftonbladetIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.search(self._VALID_URL, url)
-
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         # find internal video meta data

From f6c24009bea95104e186afd77a867abbc61b7f81 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:52:22 +0100
Subject: [PATCH 768/974] [YoutubeDL] Calculate thumbnail IDs automatically

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c18ce9660..1730df4cd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -964,9 +964,11 @@ class YoutubeDL(object):
             thumbnails.sort(key=lambda t: (
                 t.get('preference'), t.get('width'), t.get('height'),
                 t.get('id'), t.get('url')))
-            for t in thumbnails:
+            for i, t in enumerate(thumbnails):
                 if 'width' in t and 'height' in t:
                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
+                if t.get('id') is None:
+                    t['id'] = '%d' % i
 
         if thumbnails and 'thumbnail' not in info_dict:
             info_dict['thumbnail'] = thumbnails[-1]['url']

From 1a6373ef3960add0117f797e0afe3322352c1c52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:53:05 +0100
Subject: [PATCH 769/974] [sort_formats] Prefer bitrate over video size

720p @ 1000KB/s looks way better than 1080p @ 500KB/s
---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9e517e3ac..653d793fc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -704,11 +704,11 @@ class InfoExtractor(object):
                 preference,
                 f.get('language_preference') if f.get('language_preference') is not None else -1,
                 f.get('quality') if f.get('quality') is not None else -1,
-                f.get('height') if f.get('height') is not None else -1,
-                f.get('width') if f.get('width') is not None else -1,
-                ext_preference,
                 f.get('tbr') if f.get('tbr') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,
+                ext_preference,
+                f.get('height') if f.get('height') is not None else -1,
+                f.get('width') if f.get('width') is not None else -1,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('fps') if f.get('fps') is not None else -1,

From 9bb8e0a3f9276f65de38cda431bf72f7bd266693 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:58:28 +0100
Subject: [PATCH 770/974] [wsj] Add new extractor (Fixes #4854)

---
 test/test_utils.py               |  3 ++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/common.py   |  1 +
 youtube_dl/extractor/wsj.py      | 89 ++++++++++++++++++++++++++++++++
 youtube_dl/utils.py              |  2 +-
 5 files changed, 95 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/wsj.py

diff --git a/test/test_utils.py b/test/test_utils.py
index 0ffccd35f..80c765bc4 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -156,6 +156,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(
             unified_strdate('11/26/2014 11:30:00 AM PST', day_first=False),
             '20141126')
+        self.assertEqual(
+            unified_strdate('2/2/2015 6:47:40 PM', day_first=False),
+            '20150202')
 
     def test_find_xpath_attr(self):
         testxml = '''<root>
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5dcb14feb..5866a7617 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -554,6 +554,7 @@ from .wimp import WimpIE
 from .wistia import WistiaIE
 from .worldstarhiphop import WorldStarHipHopIE
 from .wrzuta import WrzutaIE
+from .wsj import WSJIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
 from .xhamster import XHamsterIE
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 653d793fc..602601b24 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -145,6 +145,7 @@ class InfoExtractor(object):
     thumbnail:      Full URL to a video thumbnail image.
     description:    Full video description.
     uploader:       Full name of the video uploader.
+    creator:        The main artist who created the video.
     timestamp:      UNIX timestamp of the moment the video became available.
     upload_date:    Video upload date (YYYYMMDD).
                     If not explicitly set, calculated from timestamp.
diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
new file mode 100644
index 000000000..cbe3dc7be
--- /dev/null
+++ b/youtube_dl/extractor/wsj.py
@@ -0,0 +1,89 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+)
+
+
+class WSJIE(InfoExtractor):
+    _VALID_URL = r'https?://video-api\.wsj\.com/api-video/player/iframe\.html\?guid=(?P<id>[a-zA-Z0-9-]+)'
+    IE_DESC = 'Wall Street Journal'
+    _TEST = {
+        'url': 'http://video-api.wsj.com/api-video/player/iframe.html?guid=1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
+        'md5': '9747d7a6ebc2f4df64b981e1dde9efa9',
+        'info_dict': {
+            'id': '1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
+            'ext': 'mp4',
+            'upload_date': '20150202',
+            'uploader_id': 'bbright',
+            'creator': 'bbright',
+            'categories': list,  # a long list
+            'duration': 90,
+            'title': 'Bills Coach Rex Ryan Updates His Old Jets Tattoo',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        bitrates = [128, 174, 264, 320, 464, 664, 1264]
+        api_url = (
+            'http://video-api.wsj.com/api-video/find_all_videos.asp?'
+            'type=guid&count=1&query=%s&'
+            'fields=hls,adZone,thumbnailList,guid,state,secondsUntilStartTime,'
+            'author,description,name,linkURL,videoStillURL,duration,videoURL,'
+            'adCategory,catastrophic,linkShortURL,doctypeID,youtubeID,'
+            'titletag,rssURL,wsj-section,wsj-subsection,allthingsd-section,'
+            'allthingsd-subsection,sm-section,sm-subsection,provider,'
+            'formattedCreationDate,keywords,keywordsOmniture,column,editor,'
+            'emailURL,emailPartnerID,showName,omnitureProgramName,'
+            'omnitureVideoFormat,linkRelativeURL,touchCastID,'
+            'omniturePublishDate,%s') % (
+                video_id, ','.join('video%dkMP4Url' % br for br in bitrates))
+        info = self._download_json(api_url, video_id)['items'][0]
+
+        # Thumbnails are conveniently in the correct format already
+        thumbnails = info.get('thumbnailList')
+        creator = info.get('author')
+        uploader_id = info.get('editor')
+        categories = info.get('keywords')
+        duration = int_or_none(info.get('duration'))
+        upload_date = unified_strdate(
+            info.get('formattedCreationDate'), day_first=False)
+        title = info.get('name', info.get('titletag'))
+
+        formats = [{
+            'format_id': 'f4m',
+            'format_note': 'f4m (meta URL)',
+            'url': info['videoURL'],
+        }]
+        if info.get('hls'):
+            formats.extend(self._extract_m3u8_formats(
+                info['hls'], video_id, ext='mp4',
+                preference=0, entry_protocol='m3u8_native'))
+        for br in bitrates:
+            field = 'video%dkMP4Url' % br
+            if info.get(field):
+                formats.append({
+                    'format_id': 'mp4-%d' % br,
+                    'container': 'mp4',
+                    'tbr': br,
+                    'url': info[field],
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'creator': creator,
+            'uploader_id': uploader_id,
+            'duration': duration,
+            'upload_date': upload_date,
+            'title': title,
+            'formats': formats,
+            'categories': categories,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 251074bf5..8f5463f1c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -701,7 +701,7 @@ def unified_strdate(date_str, day_first=True):
     # %z (UTC offset) is only supported in python>=3.2
     date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
     # Remove AM/PM + timezone
-    date_str = re.sub(r'(?i)\s*(?:AM|PM)\s+[A-Z]+', '', date_str)
+    date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
 
     format_expressions = [
         '%d %B %Y',

From cd7342755fb6928ca9f089bbd4bfdd17904f91b3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:59:27 +0100
Subject: [PATCH 771/974] release 2015.02.03.1

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0cebe3981..b188be636 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -511,6 +511,7 @@
  - **Wistia**
  - **WorldStarHipHop**
  - **wrzuta.pl**
+ - **WSJ**: Wall Street Journal
  - **XBef**
  - **XboxClips**
  - **XHamster**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a96e74629..4975ff83e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.03'
+__version__ = '2015.02.03.1'

From 12f0454cd60ffa67b57a822b2e30461cb761040d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 14:18:12 +0100
Subject: [PATCH 772/974] [README] Add an FAQ entry about anime sites

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index c1221417e..06dea400d 100644
--- a/README.md
+++ b/README.md
@@ -532,6 +532,14 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
     youtube-dl -- -wNyEUrxzFU
     youtube-dl "http://www.youtube.com/watch?v=-wNyEUrxzFU"
 
+### Can you add support for this anime video site, or site which shows current movies for free?
+
+As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.
+
+A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into youtube-dl. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization.
+
+Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content.
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.

From f076b63821100af17379ed8b470e508846bb8432 Mon Sep 17 00:00:00 2001
From: naglis <njonaitis@gmail.com>
Date: Wed, 4 Feb 2015 15:33:37 +0200
Subject: [PATCH 773/974] [generic/ooyala] Add support for Ooyala embeds on SBN
 network websites (Fixes #4859)

---
 youtube_dl/extractor/generic.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a6f102a4e..fbbc79a57 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -140,6 +140,19 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Ooyala'],
         },
+        # multiple ooyala embeds on SBN network websites
+        {
+            'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
+            'info_dict': {
+                'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
+                'title': '25 lies you will tell yourself on National Signing Day - SBNation.com',
+            },
+            'playlist_mincount': 3,
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': ['Ooyala'],
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
@@ -882,10 +895,19 @@ class GenericIE(InfoExtractor):
 
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
-                re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
+                re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
+                re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage))
         if mobj is not None:
             return OoyalaIE._build_url_result(mobj.group('ec'))
 
+        # Look for multiple Ooyala embeds on SBN network websites
+        mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
+        if mobj is not None:
+            embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
+            if embeds:
+                return _playlist_from_matches(
+                    embeds, getter=lambda v: OoyalaIE._url_for_embed_code(v['provider_video_id']), ie='Ooyala')
+
         # Look for Aparat videos
         mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
         if mobj is not None:

From 58ba6c016086f1f9f47778f7a64aca72beb55c23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Feb 2015 19:47:55 +0600
Subject: [PATCH 774/974] [mixcloud] Fix extraction (Closes #4862)

---
 youtube_dl/extractor/mixcloud.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 141bb463e..1831c6749 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -18,7 +18,7 @@ class MixcloudIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([^/]+)/([^/]+)'
     IE_NAME = 'mixcloud'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
         'info_dict': {
             'id': 'dholbach-cryptkeeper',
@@ -33,7 +33,20 @@ class MixcloudIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
         },
-    }
+    }, {
+        'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
+        'info_dict': {
+            'id': 'gillespeterson-caribou-7-inch-vinyl-mix-chat',
+            'ext': 'm4a',
+            'title': 'Electric Relaxation vol. 3',
+            'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
+            'uploader': 'Daniel Drumz',
+            'uploader_id': 'gillespeterson',
+            'thumbnail': 're:https?://.*\.jpg',
+            'view_count': int,
+            'like_count': int,
+        },
+    }]
 
     def _get_url(self, track_id, template_url):
         server_count = 30
@@ -60,7 +73,7 @@ class MixcloudIE(InfoExtractor):
         webpage = self._download_webpage(url, track_id)
 
         preview_url = self._search_regex(
-            r'\s(?:data-preview-url|m-preview)="(.+?)"', webpage, 'preview url')
+            r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url')
         song_url = preview_url.replace('/previews/', '/c/originals/')
         template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
         final_song_url = self._get_url(track_id, template_url)

From 7dfc3566250894147910bd6b79530f97770a47f3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 4 Feb 2015 16:09:35 +0100
Subject: [PATCH 775/974] release 2015.02.04

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4975ff83e..36591a43d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.03.1'
+__version__ = '2015.02.04'

From 25e5ebf382b2fca9d4b6909ac1ac9b5af89bcdd9 Mon Sep 17 00:00:00 2001
From: Frans de Jonge <fransdejonge@gmail.com>
Date: Thu, 5 Feb 2015 12:51:33 +0100
Subject: [PATCH 776/974] Add NPO.nl subtitles

Implements #3638
---
 youtube_dl/extractor/npo.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 175b14583..0580461bb 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
@@ -11,7 +11,7 @@ from ..utils import (
 )
 
 
-class NPOBaseIE(InfoExtractor):
+class NPOBaseIE(SubtitlesInfoExtractor):
     def _get_token(self, video_id):
         token_page = self._download_webpage(
             'http://ida.omroep.nl/npoplayer/i.js',
@@ -159,6 +159,13 @@ class NPOIE(NPOBaseIE):
                     'quality': stream.get('kwaliteit'),
                 })
 
+        subtitles = {}
+
+        tt888 = metadata.get('tt888')
+        if self._have_to_download_any_subtitles and tt888 == 'ja':
+            subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
+            subtitles = self.extract_subtitles(video_id, subtitles)
+
         self._sort_formats(formats)
 
         return {
@@ -169,6 +176,7 @@ class NPOIE(NPOBaseIE):
             'upload_date': unified_strdate(metadata.get('gidsdatum')),
             'duration': parse_duration(metadata.get('tijdsduur')),
             'formats': formats,
+            'subtitles': subtitles,
         }
 
 

From 9e91449c8d1dae53eda6fb250384f0afa5122ce6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 20:13:28 +0600
Subject: [PATCH 777/974] [npo] Fix subtitles (Closes #3638)

---
 youtube_dl/extractor/npo.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 0580461bb..54be06a4e 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -159,15 +159,18 @@ class NPOIE(NPOBaseIE):
                     'quality': stream.get('kwaliteit'),
                 })
 
-        subtitles = {}
-
-        tt888 = metadata.get('tt888')
-        if self._have_to_download_any_subtitles and tt888 == 'ja':
-            subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
-            subtitles = self.extract_subtitles(video_id, subtitles)
-
         self._sort_formats(formats)
 
+        subtitles = {}
+        if metadata.get('tt888') == 'ja':
+            subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
+
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
+        subtitles = self.extract_subtitles(video_id, subtitles)
+
         return {
             'id': video_id,
             'title': metadata['titel'],

From 6a348cf7d507f0373e9a6f67358202b66a858cd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 20:14:56 +0600
Subject: [PATCH 778/974] Credit @Frenzie for npo subtitles (#4878)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 83d8161a9..85ca81a27 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -108,3 +108,4 @@ Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
 Paul Hartmann
+Frans de Jonge

From 60ca389c646ae03e90377261857bfe45333a8ff2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 22:16:27 +0600
Subject: [PATCH 779/974] [extractor/common] Prefix f4m/m3u8 entries with
 identifier

---
 youtube_dl/extractor/common.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 602601b24..8ec7cee06 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -765,7 +765,7 @@ class InfoExtractor(object):
         self.to_screen(msg)
         time.sleep(timeout)
 
-    def _extract_f4m_formats(self, manifest_url, video_id):
+    def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest')
@@ -780,24 +780,25 @@ class InfoExtractor(object):
             if manifest_version == '2.0':
                 manifest_url = '/'.join(manifest_url.split('/')[:-1]) + '/' + media_el.attrib.get('href')
             tbr = int_or_none(media_el.attrib.get('bitrate'))
-            format_id = 'f4m-%d' % (i if tbr is None else tbr)
             formats.append({
-                'format_id': format_id,
+                'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),
                 'url': manifest_url,
                 'ext': 'flv',
                 'tbr': tbr,
                 'width': int_or_none(media_el.attrib.get('width')),
                 'height': int_or_none(media_el.attrib.get('height')),
+                'preference': preference,
             })
         self._sort_formats(formats)
 
         return formats
 
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
-                              entry_protocol='m3u8', preference=None):
+                              entry_protocol='m3u8', preference=None,
+                              m3u8_id=None):
 
         formats = [{
-            'format_id': 'm3u8-meta',
+            'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-meta'])),
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',
@@ -833,9 +834,8 @@ class InfoExtractor(object):
                     formats.append({'url': format_url(line)})
                     continue
                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
-
                 f = {
-                    'format_id': 'm3u8-%d' % (tbr if tbr else len(formats)),
+                    'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))])),
                     'url': format_url(line.strip()),
                     'tbr': tbr,
                     'ext': ext,

From f13b88c61696eaeb96878b5c68b7ab949a3f4ca6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 22:17:50 +0600
Subject: [PATCH 780/974] [rts] Fix f4m and m3u8 extraction (Closes #4873)

---
 youtube_dl/extractor/rts.py | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index 5e84c1098..d0981115d 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -6,12 +6,14 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
+    compat_urllib_parse_urlparse,
 )
 from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
     unescapeHTML,
+    xpath_text,
 )
 
 
@@ -159,11 +161,27 @@ class RTSIE(InfoExtractor):
             return int_or_none(self._search_regex(
                 r'-([0-9]+)k\.', url, 'bitrate', default=None))
 
-        formats = [{
-            'format_id': fid,
-            'url': furl,
-            'tbr': extract_bitrate(furl),
-        } for fid, furl in info['streams'].items()]
+        formats = []
+        for format_id, format_url in info['streams'].items():
+            if format_url.endswith('.f4m'):
+                token = self._download_xml(
+                    'http://tp.srgssr.ch/token/akahd.xml?stream=%s/*' % compat_urllib_parse_urlparse(format_url).path,
+                    video_id, 'Downloading %s token' % format_id)
+                auth_params = xpath_text(token, './/authparams', 'auth params')
+                if not auth_params:
+                    continue
+                formats.extend(self._extract_f4m_formats(
+                    '%s?%s&hdcore=3.4.0&plugin=aasp-3.4.0.132.66' % (format_url, auth_params),
+                    video_id, f4m_id=format_id))
+            elif format_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id))
+            else:
+                formats.append({
+                    'format_id': format_id,
+                    'url': format_url,
+                    'tbr': extract_bitrate(format_url),
+                })
 
         if 'media' in info:
             formats.extend([{

From 300753a0695c17979d6ac42147cb162ff0eedea5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 00:51:16 +0600
Subject: [PATCH 781/974] [YoutubeDL] Fix video+audio format field (Closes
 #4880)

---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1730df4cd..9605f8f99 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1076,7 +1076,8 @@ class YoutubeDL(object):
                                 else self.params['merge_output_format'])
                             selected_format = {
                                 'requested_formats': formats_info,
-                                'format': rf,
+                                'format': '%s+%s' % (formats_info[0].get('format'),
+                                                     formats_info[1].get('format')),
                                 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
                                                         formats_info[1].get('format_id')),
                                 'width': formats_info[0].get('width'),

From e3aaace4005e7b767d28152e60d1cf6272c398ec Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 19:55:41 +0100
Subject: [PATCH 782/974] [tweakers] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tweakers.py | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/tweakers.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5866a7617..047f7002a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -475,6 +475,7 @@ from .tutv import TutvIE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
+from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twitch import (
     TwitchVideoIE,
diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
new file mode 100644
index 000000000..3080f9e46
--- /dev/null
+++ b/youtube_dl/extractor/tweakers.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class TweakersIE(InfoExtractor):
+    _VALID_URL = r'https?://tweakers\.net/video/(?P<id>[0-9]+).*'
+    _TEST = {
+        'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
+        'md5': 'f7f7f3027166a7f32f024b4ae6571ced',
+        'info_dict': {
+            'id': '9926',
+            'ext': 'mp4',
+            'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
+            # TODO more properties, either as:
+            # * A value
+            # * MD5 checksum; start the string with md5:
+            # * A regular expression; start the string with re:
+            # * Any Python type (for example int or float)
+        }
+    }
+
+    def _real_extract(self, url):
+        splitted_url = re.split('.html|/', url)
+        del splitted_url[-1] # To remove extra '/' at the end
+        video_id = splitted_url[4]
+        title = splitted_url[5].title() # Retrieve title for URL and capitalize
+        splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page
+        player_url = '/'.join(splitted_url) + '.html'
+        player_page = self._download_webpage(player_url, video_id)
+
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'title': title,
+            'url':  re.findall('http.*mp4', player_page)[0],
+            'player_url': player_url
+        }

From 3ee6e02564f7f6c54480fd4eeb94ae08a747b9b6 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 19:59:36 +0100
Subject: [PATCH 783/974] Edit Tweakers extractor

Fixed code conventions (mainly adding two or more spaces before making
an inline comment)
---
 youtube_dl/extractor/tweakers.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 3080f9e46..29cd04f00 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -25,10 +25,10 @@ class TweakersIE(InfoExtractor):
 
     def _real_extract(self, url):
         splitted_url = re.split('.html|/', url)
-        del splitted_url[-1] # To remove extra '/' at the end
+        del splitted_url[-1]  # To remove extra '/' at the end
         video_id = splitted_url[4]
-        title = splitted_url[5].title() # Retrieve title for URL and capitalize
-        splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page
+        title = splitted_url[5].title()  # Retrieve title for URL and capitalize
+        splitted_url[3] = splitted_url[3] + '/player'  # Add /player to get the player page
         player_url = '/'.join(splitted_url) + '.html'
         player_page = self._download_webpage(player_url, video_id)
 
@@ -36,6 +36,6 @@ class TweakersIE(InfoExtractor):
             'id': video_id,
             'ext': 'mp4',
             'title': title,
-            'url':  re.findall('http.*mp4', player_page)[0],
+            'url': re.findall('http.*mp4', player_page)[0],
             'player_url': player_url
         }

From c893d70805be9b8f8a10c7d7dbe39216f8976ea5 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:38:35 +0100
Subject: [PATCH 784/974] Remove player-url in tweakers.py

Player-url only needed for rmftp, not for regular URLs
---
 youtube_dl/extractor/tweakers.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 29cd04f00..14145e617 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -37,5 +37,4 @@ class TweakersIE(InfoExtractor):
             'ext': 'mp4',
             'title': title,
             'url': re.findall('http.*mp4', player_page)[0],
-            'player_url': player_url
         }

From bc63d56ccac8805ca8394e0b41890027e6d677d2 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:40:18 +0100
Subject: [PATCH 785/974] Remove unnecessary TODO-comments

---
 youtube_dl/extractor/tweakers.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 14145e617..a4e749850 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -15,11 +15,6 @@ class TweakersIE(InfoExtractor):
             'id': '9926',
             'ext': 'mp4',
             'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
-            # TODO more properties, either as:
-            # * A value
-            # * MD5 checksum; start the string with md5:
-            # * A regular expression; start the string with re:
-            # * Any Python type (for example int or float)
         }
     }
 

From 67ce4f8820f94961c9b094f9c652f421c9d9f6fb Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:49:13 +0100
Subject: [PATCH 786/974] Use match_id method instead of splitted URL

---
 youtube_dl/extractor/tweakers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index a4e749850..e332d4694 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -21,7 +21,7 @@ class TweakersIE(InfoExtractor):
     def _real_extract(self, url):
         splitted_url = re.split('.html|/', url)
         del splitted_url[-1]  # To remove extra '/' at the end
-        video_id = splitted_url[4]
+        video_id = self._match_id(url)
         title = splitted_url[5].title()  # Retrieve title for URL and capitalize
         splitted_url[3] = splitted_url[3] + '/player'  # Add /player to get the player page
         player_url = '/'.join(splitted_url) + '.html'

From 9c91a8fa70e3c2f29dd23809e90b3357b6c1cdc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 03:23:42 +0600
Subject: [PATCH 787/974] [tweakers] Switch extraction to xspf playlist,
 extract all formats and meta (#4881)

---
 youtube_dl/extractor/tweakers.py | 58 ++++++++++++++++++++++++--------
 1 file changed, 44 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index e332d4694..c80ec15cf 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -1,35 +1,65 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    xpath_with_ns,
+    int_or_none,
+    float_or_none,
+)
 
 
 class TweakersIE(InfoExtractor):
-    _VALID_URL = r'https?://tweakers\.net/video/(?P<id>[0-9]+).*'
+    _VALID_URL = r'https?://tweakers\.net/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
-        'md5': 'f7f7f3027166a7f32f024b4ae6571ced',
+        'md5': '1b5afa817403bb5baa08359dca31e6df',
         'info_dict': {
             'id': '9926',
             'ext': 'mp4',
-            'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
+            'title': 'New Nintendo 3DS XL - Op alle fronten beter',
+            'description': 'md5:f97324cc71e86e11c853f0763820e3ba',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
+            'duration': 386,
         }
     }
 
     def _real_extract(self, url):
-        splitted_url = re.split('.html|/', url)
-        del splitted_url[-1]  # To remove extra '/' at the end
         video_id = self._match_id(url)
-        title = splitted_url[5].title()  # Retrieve title for URL and capitalize
-        splitted_url[3] = splitted_url[3] + '/player'  # Add /player to get the player page
-        player_url = '/'.join(splitted_url) + '.html'
-        player_page = self._download_webpage(player_url, video_id)
+
+        playlist = self._download_xml(
+            'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % video_id,
+            video_id)
+
+        NS_MAP = {
+            'xspf': 'http://xspf.org/ns/0/',
+            's1': 'http://static.streamone.nl/player/ns/0',
+        }
+
+        track = playlist.find(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP))
+
+        title = xpath_text(
+            track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
+        description = xpath_text(
+            track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
+        thumbnail = xpath_text(
+            track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
+        duration = float_or_none(
+            xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'),
+            1000)
+
+        formats = [{
+            'url': location.text,
+            'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
+            'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
+            'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
+        } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
 
         return {
             'id': video_id,
-            'ext': 'mp4',
             'title': title,
-            'url': re.findall('http.*mp4', player_page)[0],
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
         }

From 3900eec27c0a385d60f2b1dca50faa6f5e25b971 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 04:29:29 +0600
Subject: [PATCH 788/974] [extractor/common] Fix 2.0 manifest extraction
 (Closes #4830)

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8ec7cee06..df1a4417b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -778,7 +778,8 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                manifest_url = '/'.join(manifest_url.split('/')[:-1]) + '/' + media_el.attrib.get('href')
+                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/'
+                                + (media_el.attrib.get('href') or media_el.attrib.get('url')))
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),

From 1a2548d9e91b24f60d6948e658cfc0b68797b254 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 6 Feb 2015 13:41:58 +0200
Subject: [PATCH 789/974] [rtp] Pass --realtime to rtmpdump (Fixes #4882)

A workaround for video jumping back in time.
---
 youtube_dl/extractor/rtp.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index 7736cabba..4511cba49 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -49,6 +49,7 @@ class RTPIE(InfoExtractor):
             'ext': ext,
             'vcodec': config.get('type') == 'audio' and 'none' or None,
             'player_url': 'http://programas.rtp.pt/play/player.swf?v3',
+            'rtmp_real_time': True,
         }]
 
         return {

From c8319733667ff5bc0e0cbe0a9c253c8cfc7d479e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Feb 2015 14:38:30 +0100
Subject: [PATCH 790/974] release 2015.02.06

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b188be636..2d8f9c316 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -441,6 +441,7 @@
  - **tvp.pl**
  - **tvp.pl:Series**
  - **TVPlay**: TV3Play and related services
+ - **Tweakers**
  - **twitch:bookmarks**
  - **twitch:chapter**
  - **twitch:past_broadcasts**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 36591a43d..1091ae61b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.04'
+__version__ = '2015.02.06'

From 1e1896f2de300bf1905a63adffe17e7ab68cc252 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Feb 2015 15:16:43 +0100
Subject: [PATCH 791/974] [extractor/common] Correct sort order.

We should look at height and width before ext_preference.
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index df1a4417b..980b63680 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -707,9 +707,9 @@ class InfoExtractor(object):
                 f.get('quality') if f.get('quality') is not None else -1,
                 f.get('tbr') if f.get('tbr') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,
-                ext_preference,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
+                ext_preference,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('fps') if f.get('fps') is not None else -1,

From 7f2a9f1b4993693659820693c1b6432f3b01d21b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 21:15:01 +0600
Subject: [PATCH 792/974] [tvigle] Add support for cloud URLs (Closes #4887)

---
 youtube_dl/extractor/tvigle.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index ba65996dc..102362b29 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -1,6 +1,8 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
@@ -11,7 +13,7 @@ from ..utils import (
 class TvigleIE(InfoExtractor):
     IE_NAME = 'tvigle'
     IE_DESC = 'Интернет-телевидение Tvigle.ru'
-    _VALID_URL = r'http://(?:www\.)?tvigle\.ru/(?:[^/]+/)+(?P<id>[^/]+)/$'
+    _VALID_URL = r'https?://(?:www\.)?(?:tvigle\.ru/(?:[^/]+/)+(?P<display_id>[^/]+)/$|cloud\.tvigle\.ru/video/(?P<id>\d+))'
 
     _TESTS = [
         {
@@ -38,16 +40,22 @@ class TvigleIE(InfoExtractor):
                 'duration': 186.080,
                 'age_limit': 0,
             },
-        },
+        }, {
+            'url': 'https://cloud.tvigle.ru/video/5267604/',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, display_id)
-
-        video_id = self._html_search_regex(
-            r'<li class="video-preview current_playing" id="(\d+)">', webpage, 'video id')
+        if not video_id:
+            webpage = self._download_webpage(url, display_id)
+            video_id = self._html_search_regex(
+                r'<li class="video-preview current_playing" id="(\d+)">',
+                webpage, 'video id')
 
         video_data = self._download_json(
             'http://cloud.tvigle.ru/api/play/video/%s/' % video_id, display_id)

From d6eb66ed3c0dc1362c1fec46b2d9a60076812f50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Feb 2015 01:46:54 +0600
Subject: [PATCH 793/974] [aftenposten] Add extractor (Closes #4863)

---
 youtube_dl/extractor/__init__.py    |   1 +
 youtube_dl/extractor/aftenposten.py | 103 ++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)
 create mode 100644 youtube_dl/extractor/aftenposten.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 047f7002a..0d7a120bc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -6,6 +6,7 @@ from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
 from .adobetv import AdobeTVIE
 from .adultswim import AdultSwimIE
+from .aftenposten import AftenpostenIE
 from .aftonbladet import AftonbladetIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py
new file mode 100644
index 000000000..2b257ede7
--- /dev/null
+++ b/youtube_dl/extractor/aftenposten.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    xpath_with_ns,
+    xpath_text,
+    find_xpath_attr,
+)
+
+
+class AftenpostenIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/([^/]+/)*(?P<id>[^/]+)-\d+\.html'
+
+    _TEST = {
+        'url': 'http://www.aftenposten.no/webtv/serier-og-programmer/sweatshopenglish/TRAILER-SWEATSHOP---I-cant-take-any-more-7800835.html?paging=&section=webtv_serierogprogrammer_sweatshop_sweatshopenglish',
+        'md5': 'fd828cd29774a729bf4d4425fe192972',
+        'info_dict': {
+            'id': '21039',
+            'ext': 'mov',
+            'title': 'TRAILER: "Sweatshop" - I can´t take any more',
+            'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
+            'timestamp': 1416927969,
+            'upload_date': '20141125',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._html_search_regex(
+            r'data-xs-id="(\d+)"', webpage, 'video id')
+
+        data = self._download_xml(
+            'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=%s' % video_id, video_id)
+
+        NS_MAP = {
+            'atom': 'http://www.w3.org/2005/Atom',
+            'xt': 'http://xstream.dk/',
+            'media': 'http://search.yahoo.com/mrss/',
+        }
+
+        entry = data.find(xpath_with_ns('./atom:entry', NS_MAP))
+
+        title = xpath_text(
+            entry, xpath_with_ns('./atom:title', NS_MAP), 'title')
+        description = xpath_text(
+            entry, xpath_with_ns('./atom:summary', NS_MAP), 'description')
+        timestamp = parse_iso8601(xpath_text(
+            entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date'))
+
+        formats = []
+        media_group = entry.find(xpath_with_ns('./media:group', NS_MAP))
+        for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)):
+            media_url = media_content.get('url')
+            if not media_url:
+                continue
+            tbr = int_or_none(media_content.get('bitrate'))
+            mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', media_url)
+            if mobj:
+                formats.append({
+                    'url': mobj.group('url'),
+                    'play_path': 'mp4:%s' % mobj.group('playpath'),
+                    'app': mobj.group('app'),
+                    'ext': 'flv',
+                    'tbr': tbr,
+                    'format_id': 'rtmp-%d' % tbr,
+                })
+            else:
+                formats.append({
+                    'url': media_url,
+                    'tbr': tbr,
+                })
+        self._sort_formats(formats)
+
+        link = find_xpath_attr(
+            entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original')
+        if link is not None:
+            formats.append({
+                'url': link.get('href'),
+                'format_id': link.get('rel'),
+            })
+
+        thumbnails = [{
+            'url': splash.get('url'),
+            'width': int_or_none(splash.get('width')),
+            'height': int_or_none(splash.get('height')),
+        } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }

From ad5747bad146c76f545870911d5f2fb586c0f7b1 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 6 Feb 2015 22:59:17 +0200
Subject: [PATCH 794/974] [rtp] Construct regular HTTP download URLs (#4882)

---
 youtube_dl/extractor/rtp.py | 42 +++++++++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index 4511cba49..ecf4939cd 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -1,16 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
+import re
 
 from .common import InfoExtractor
-from ..utils import js_to_json
 
 
 class RTPIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/(?P<id>[^/?#]+)/?'
     _TESTS = [{
         'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
+        'md5': 'e736ce0c665e459ddb818546220b4ef8',
         'info_dict': {
             'id': 'e174042',
             'ext': 'mp3',
@@ -18,9 +18,6 @@ class RTPIE(InfoExtractor):
             'description': 'As paixões musicais de António Cartaxo e António Macedo',
             'thumbnail': 're:^https?://.*\.jpg',
         },
-        'params': {
-            'skip_download': True,  # RTMP download
-        },
     }, {
         'url': 'http://www.rtp.pt/play/p831/a-quimica-das-coisas',
         'only_matching': True,
@@ -37,21 +34,48 @@ class RTPIE(InfoExtractor):
 
         player_config = self._search_regex(
             r'(?s)RTPPLAY\.player\.newPlayer\(\s*(\{.*?\})\s*\)', webpage, 'player config')
-        config = json.loads(js_to_json(player_config))
+        config = self._parse_json(player_config, video_id)
 
         path, ext = config.get('file').rsplit('.', 1)
         formats = [{
+            'format_id': 'rtmp',
+            'ext': ext,
+            'vcodec': config.get('type') == 'audio' and 'none' or None,
+            'preference': -2,
+            'url': 'rtmp://{streamer:s}/{application:s}'.format(**config),
             'app': config.get('application'),
             'play_path': '{ext:s}:{path:s}'.format(ext=ext, path=path),
             'page_url': url,
-            'url': 'rtmp://{streamer:s}/{application:s}'.format(**config),
             'rtmp_live': config.get('live', False),
-            'ext': ext,
-            'vcodec': config.get('type') == 'audio' and 'none' or None,
             'player_url': 'http://programas.rtp.pt/play/player.swf?v3',
             'rtmp_real_time': True,
         }]
 
+        # Construct regular HTTP download URLs
+        replacements = {
+            'audio': {
+                'format_id': 'mp3',
+                'pattern': r'^nas2\.share/wavrss/',
+                'repl': 'http://rsspod.rtp.pt/podcasts/',
+                'vcodec': 'none',
+            },
+            'video': {
+                'format_id': 'mp4_h264',
+                'pattern': r'^nas2\.share/h264/',
+                'repl': 'http://rsspod.rtp.pt/videocasts/',
+                'vcodec': 'h264',
+            },
+        }
+        r = replacements[config['type']]
+        if re.match(r['pattern'], config['file']) is not None:
+            formats.append({
+                'format_id': r['format_id'],
+                'url': re.sub(r['pattern'], r['repl'], config['file']),
+                'vcodec': r['vcodec'],
+            })
+
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
             'title': title,

From 467d3c9a0c364fe4e24c6efd1d20317a80eee547 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 22:05:11 +0100
Subject: [PATCH 795/974] [ffmpeg] --extrac-audio: Use the same options for
 avconv and ffmpeg

They have been available in ffmpeg since version 0.9, and we require 1.0 or higher.
---
 youtube_dl/postprocessor/ffmpeg.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4a4422c5a..01d25f760 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -166,14 +166,13 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         if filecodec is None:
             raise PostProcessingError('WARNING: unable to obtain file audio codec with ffprobe')
 
-        uses_avconv = self._uses_avconv()
         more_opts = []
         if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
             if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']:
                 # Lossless, but in another container
                 acodec = 'copy'
                 extension = 'm4a'
-                more_opts = ['-bsf:a' if uses_avconv else '-absf', 'aac_adtstoasc']
+                more_opts = ['-bsf:a', 'aac_adtstoasc']
             elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
                 # Lossless if possible
                 acodec = 'copy'
@@ -189,9 +188,9 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 more_opts = []
                 if self._preferredquality is not None:
                     if int(self._preferredquality) < 10:
-                        more_opts += ['-q:a' if uses_avconv else '-aq', self._preferredquality]
+                        more_opts += ['-q:a', self._preferredquality]
                     else:
-                        more_opts += ['-b:a' if uses_avconv else '-ab', self._preferredquality + 'k']
+                        more_opts += ['-b:a', self._preferredquality + 'k']
         else:
             # We convert the audio (lossy)
             acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
@@ -200,13 +199,13 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             if self._preferredquality is not None:
                 # The opus codec doesn't support the -aq option
                 if int(self._preferredquality) < 10 and extension != 'opus':
-                    more_opts += ['-q:a' if uses_avconv else '-aq', self._preferredquality]
+                    more_opts += ['-q:a', self._preferredquality]
                 else:
-                    more_opts += ['-b:a' if uses_avconv else '-ab', self._preferredquality + 'k']
+                    more_opts += ['-b:a', self._preferredquality + 'k']
             if self._preferredcodec == 'aac':
                 more_opts += ['-f', 'adts']
             if self._preferredcodec == 'm4a':
-                more_opts += ['-bsf:a' if uses_avconv else '-absf', 'aac_adtstoasc']
+                more_opts += ['-bsf:a', 'aac_adtstoasc']
             if self._preferredcodec == 'vorbis':
                 extension = 'ogg'
             if self._preferredcodec == 'wav':

From b3a286d69d2bf28ba32a77b0fe66932eb6d1bc65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 22:23:06 +0100
Subject: [PATCH 796/974] [YoutubeDL] _calc_cookies: add get_header method to
 _PseudoRequest (#4861)

---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9605f8f99..633e3d8a1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -938,6 +938,9 @@ class YoutubeDL(object):
             def has_header(self, h):
                 return h in self.headers
 
+            def get_header(self, h, default=None):
+                return self.headers.get(h, default)
+
         pr = _PseudoRequest(info_dict['url'])
         self.cookiejar.add_cookie_header(pr)
         return pr.headers.get('Cookie')

From 2b4ecde2c8e7d405758a3a80190c646940fe7e9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 23:54:25 +0100
Subject: [PATCH 797/974] [test/YoutubeDL] Add a simple test for postprocesors

Just checks that the 'keepvideo' option works as intended.
---
 test/test_YoutubeDL.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 678b9f7d1..47ab102a9 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -13,6 +13,7 @@ import copy
 from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
 from youtube_dl.extractor import YoutubeIE
+from youtube_dl.postprocessor.common import PostProcessor
 
 
 class YDL(FakeYDL):
@@ -370,5 +371,36 @@ class TestFormatSelection(unittest.TestCase):
             'vbr': 10,
         }), '^\s*10k$')
 
+    def test_postprocessors(self):
+        filename = 'post-processor-testfile.mp4'
+        audiofile = filename + '.mp3'
+
+        class SimplePP(PostProcessor):
+            def run(self, info):
+                print(audiofile)
+                with open(audiofile, 'wt') as f:
+                    f.write('EXAMPLE')
+                info['filepath']
+                return False, info
+
+        def run_pp(params):
+            with open(filename, 'wt') as f:
+                f.write('EXAMPLE')
+            ydl = YoutubeDL(params)
+            ydl.add_post_processor(SimplePP())
+            ydl.post_process(filename, {'filepath': filename})
+
+        run_pp({'keepvideo': True})
+        self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
+        self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
+        os.unlink(filename)
+        os.unlink(audiofile)
+
+        run_pp({'keepvideo': False})
+        self.assertFalse(os.path.exists(filename), '%s exists' % filename)
+        self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
+        os.unlink(audiofile)
+
+
 if __name__ == '__main__':
     unittest.main()

From 0161353d7d2d0f0fce1ecaa2f77471d787ee9bdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 23:58:01 +0100
Subject: [PATCH 798/974] [test/test_YoutubeDL] Remove debug print call

---
 test/test_YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 47ab102a9..b1cd6a69f 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -377,7 +377,6 @@ class TestFormatSelection(unittest.TestCase):
 
         class SimplePP(PostProcessor):
             def run(self, info):
-                print(audiofile)
                 with open(audiofile, 'wt') as f:
                     f.write('EXAMPLE')
                 info['filepath']

From c0e46412e99ef87f5730326ab8fa4a56038c43d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 17:30:29 +0600
Subject: [PATCH 799/974] [aparat] Fix extraction (Closes #4897)

---
 youtube_dl/extractor/aparat.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 15006336f..5d70758a3 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -34,7 +34,8 @@ class AparatIE(InfoExtractor):
                      video_id + '/vt/frame')
         webpage = self._download_webpage(embed_url, video_id)
 
-        video_urls = re.findall(r'fileList\[[0-9]+\]\s*=\s*"([^"]+)"', webpage)
+        video_urls = [video_url.replace('\\/', '/') for video_url in re.findall(
+            r'(?:fileList\[[0-9]+\]\s*=|"file"\s*:)\s*"([^"]+)"', webpage)]
         for i, video_url in enumerate(video_urls):
             req = HEADRequest(video_url)
             res = self._request_webpage(
@@ -46,7 +47,7 @@ class AparatIE(InfoExtractor):
 
         title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, 'title')
         thumbnail = self._search_regex(
-            r'\s+image:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
+            r'image:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,

From 77d2b106ccf81e50cf41a7aa0bb320433a84e110 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 13:42:41 +0100
Subject: [PATCH 800/974] [Gamekings] Fix 404 when large isn't available

When trying to download some GameKings videos, not all worked. This was
because not all videos had a "/large"-URL available. The extractor
checks now if the /large URL is available, if it isn't, it tries to get
the normal URL.
---
 youtube_dl/extractor/gamekings.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index cf8e90d7d..bac325bd8 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -31,6 +31,8 @@ class GamekingsIE(InfoExtractor):
 
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
+        if not (self._is_valid_url(video_url, video_id)):
+            video_url = video_url.replace(video_id + '/large', video_id)
 
         return {
             'id': video_id,

From f23a3ca69975346d5fc30ee09b0cdceb4a384879 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 13:47:27 +0100
Subject: [PATCH 801/974] [Gamekings] Fixed typo in URL replacement

---
 youtube_dl/extractor/gamekings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index bac325bd8..1697f5e44 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -32,7 +32,7 @@ class GamekingsIE(InfoExtractor):
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
         if not (self._is_valid_url(video_url, video_id)):
-            video_url = video_url.replace(video_id + '/large', video_id)
+            video_url = video_url.replace(video_id + 'large/', video_id)
 
         return {
             'id': video_id,

From 2f38289b79ed6c265f2e7cc91e417d0f2178371a Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 13:49:32 +0100
Subject: [PATCH 802/974] [Gamekings] Fix order of replacement string

Oops.
---
 youtube_dl/extractor/gamekings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 1697f5e44..a5d854ade 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -32,7 +32,7 @@ class GamekingsIE(InfoExtractor):
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
         if not (self._is_valid_url(video_url, video_id)):
-            video_url = video_url.replace(video_id + 'large/', video_id)
+            video_url = video_url.replace('large/' + video_id, video_id)
 
         return {
             'id': video_id,

From ba322d82090bd1126774e772b699283121ffa4b8 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 14:23:37 +0100
Subject: [PATCH 803/974] [Gamekings] Added test and replaced video_url

Quick and dirty fix for the Gamekings extractor. It gives an error about
the video_url, but it downloads it now instead of giving a 404 error on
newer Gamekings videos
---
 youtube_dl/extractor/gamekings.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index a5d854ade..aa589390a 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -7,7 +7,8 @@ from .common import InfoExtractor
 
 class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
-    _TEST = {
+    _TESTS = [
+        {
         'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
         # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
@@ -16,8 +17,17 @@ class GamekingsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review',
             'description': 'md5:36fd701e57e8c15ac8682a2374c99731',
+            }
+        },
+        {
+        'url': 'http://www.gamekings.tv/videos/the-legend-of-zelda-majoras-mask/',
+        'info_dict': {
+            'id': '118933752',
+            'ext': 'mp4',
+            'title': 'The Legend of Zelda: Majora’s Mask'
+            }
         }
-    }
+    ]
 
     def _real_extract(self, url):
 
@@ -33,6 +43,7 @@ class GamekingsIE(InfoExtractor):
         video_url = video_url.replace(video_id, 'large/' + video_id)
         if not (self._is_valid_url(video_url, video_id)):
             video_url = video_url.replace('large/' + video_id, video_id)
+            video_url = video_url.replace('http://stream.gamekings.tv/', '')
 
         return {
             'id': video_id,

From 8ca8cbe2bd262676cbe5a63aaee6559dfa795634 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 14:41:14 +0100
Subject: [PATCH 804/974] [Gamekings] Check string for vimeo, fix test

The test now doesn't fail anymore. It just checks the string for having
"vimeo" in it, instead of using the method for URL-checking, since it's
returns an error.

The tests don't fail, and the extractor works fine now.
---
 youtube_dl/extractor/gamekings.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index aa589390a..ba4d2e082 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -24,7 +24,8 @@ class GamekingsIE(InfoExtractor):
         'info_dict': {
             'id': '118933752',
             'ext': 'mp4',
-            'title': 'The Legend of Zelda: Majora’s Mask'
+            'title': 'The Legend of Zelda: Majora’s Mask',
+            'description': 'md5:9917825fe0e9f4057601fe1e38860de3'
             }
         }
     ]
@@ -41,7 +42,7 @@ class GamekingsIE(InfoExtractor):
 
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
-        if not (self._is_valid_url(video_url, video_id)):
+        if "vimeo" in video_url:
             video_url = video_url.replace('large/' + video_id, video_id)
             video_url = video_url.replace('http://stream.gamekings.tv/', '')
 

From 5d678df64a7a4a695da769f7691c86f39fca26bb Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 15:34:37 +0100
Subject: [PATCH 805/974] [Gamekings] Download playlist

Todo: URL and Thumbnail should be extracted with XPath
---
 youtube_dl/extractor/gamekings.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index ba4d2e082..f2760d444 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-
+from ..utils import xpath_text
 
 class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
@@ -35,6 +35,17 @@ class GamekingsIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
         webpage = self._download_webpage(url, name)
+
+        playlist_id = re.search(r'(?:gogoVideo)\(\d+,"?(?P<playlist_id>.*)"', webpage, re.MULTILINE).group('playlist_id')
+        playlist_url = 'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=' + playlist_id
+        playlist_rss = self._download_xml(playlist_url, playlist_id)
+
+        NS_MAP {
+            'rss': 'http://rss.jwpcdn.com/'
+         }
+
+        # Todo: Implement Xpath for searching the video link
+        
         video_url = self._og_search_video_url(webpage)
 
         video = re.search(r'[0-9]+', video_url)

From a14292e8480f339bdbf9e52e93460aec5c31ef13 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Feb 2015 16:57:07 +0200
Subject: [PATCH 806/974] [soulanime] Remove extractor (#4554)

Was supposed to be deleted by 67c2bcd
---
 youtube_dl/extractor/soulanime.py | 80 -------------------------------
 1 file changed, 80 deletions(-)
 delete mode 100644 youtube_dl/extractor/soulanime.py

diff --git a/youtube_dl/extractor/soulanime.py b/youtube_dl/extractor/soulanime.py
deleted file mode 100644
index feef33e27..000000000
--- a/youtube_dl/extractor/soulanime.py
+++ /dev/null
@@ -1,80 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    HEADRequest,
-    urlhandle_detect_ext,
-)
-
-
-class SoulAnimeWatchingIE(InfoExtractor):
-    IE_NAME = "soulanime:watching"
-    IE_DESC = "SoulAnime video"
-    _TEST = {
-        'url': 'http://www.soul-anime.net/watching/seirei-tsukai-no-blade-dance-episode-9/',
-        'md5': '05fae04abf72298098b528e98abf4298',
-        'info_dict': {
-            'id': 'seirei-tsukai-no-blade-dance-episode-9',
-            'ext': 'mp4',
-            'title': 'seirei-tsukai-no-blade-dance-episode-9',
-            'description': 'seirei-tsukai-no-blade-dance-episode-9'
-        }
-    }
-    _VALID_URL = r'http://[w.]*soul-anime\.(?P<domain>[^/]+)/watch[^/]*/(?P<id>[^/]+)'
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        domain = mobj.group('domain')
-
-        page = self._download_webpage(url, video_id)
-
-        video_url_encoded = self._html_search_regex(
-            r'<div id="download">[^<]*<a href="(?P<url>[^"]+)"', page, 'url')
-        video_url = "http://www.soul-anime." + domain + video_url_encoded
-
-        ext_req = HEADRequest(video_url)
-        ext_handle = self._request_webpage(
-            ext_req, video_id, note='Determining extension')
-        ext = urlhandle_detect_ext(ext_handle)
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'ext': ext,
-            'title': video_id,
-            'description': video_id
-        }
-
-
-class SoulAnimeSeriesIE(InfoExtractor):
-    IE_NAME = "soulanime:series"
-    IE_DESC = "SoulAnime Series"
-
-    _VALID_URL = r'http://[w.]*soul-anime\.(?P<domain>[^/]+)/anime./(?P<id>[^/]+)'
-
-    _EPISODE_REGEX = r'<option value="(/watch[^/]*/[^"]+)">[^<]*</option>'
-
-    _TEST = {
-        'url': 'http://www.soul-anime.net/anime1/black-rock-shooter-tv/',
-        'info_dict': {
-            'id': 'black-rock-shooter-tv'
-        },
-        'playlist_count': 8
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        series_id = mobj.group('id')
-        domain = mobj.group('domain')
-
-        pattern = re.compile(self._EPISODE_REGEX)
-
-        page = self._download_webpage(url, series_id, "Downloading series page")
-        mobj = pattern.findall(page)
-
-        entries = [self.url_result("http://www.soul-anime." + domain + obj) for obj in mobj]
-
-        return self.playlist_result(entries, series_id)

From 69319969de40f62ccf1c471427d309b6555483f9 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Feb 2015 17:39:00 +0200
Subject: [PATCH 807/974] [extractor/common] Add new helper method
 _family_friendly_search

---
 youtube_dl/extractor/common.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 980b63680..2f5ba7aee 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -656,6 +656,21 @@ class InfoExtractor(object):
         }
         return RATING_TABLE.get(rating.lower(), None)
 
+    def _family_friendly_search(self, html):
+        # See http://schema.org/VideoObj
+        family_friendly = self._html_search_meta('isFamilyFriendly', html)
+
+        if not family_friendly:
+            return None
+
+        RATING_TABLE = {
+            '1': 0,
+            'true': 0,
+            '0': 18,
+            'false': 18,
+        }
+        return RATING_TABLE.get(family_friendly.lower(), None)
+
     def _twitter_search_player(self, html):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')

From 955c5505e7df737b452d54d869958a8f5425edf5 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 16:44:25 +0100
Subject: [PATCH 808/974] [Gamekings] Use xpath

XPath is used for extracting the video url and the thumbnail
---
 youtube_dl/extractor/gamekings.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index f2760d444..3c1f7f195 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -3,7 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import xpath_text
+from ..utils import (
+    xpath_text,
+    xpath_with_ns
+ )
+
 
 class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
@@ -39,23 +43,22 @@ class GamekingsIE(InfoExtractor):
         playlist_id = re.search(r'(?:gogoVideo)\(\d+,"?(?P<playlist_id>.*)"', webpage, re.MULTILINE).group('playlist_id')
         playlist_url = 'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=' + playlist_id
         playlist_rss = self._download_xml(playlist_url, playlist_id)
+        
 
-        NS_MAP {
-            'rss': 'http://rss.jwpcdn.com/'
+        NS_MAP = {
+            'jwplayer': 'http://rss.jwpcdn.com/'
          }
 
-        # Todo: Implement Xpath for searching the video link
+        item = playlist_rss.find('./channel/item')
         
-        video_url = self._og_search_video_url(webpage)
-
+        image = xpath_text(item, xpath_with_ns('./jwplayer:image', NS_MAP), 'image')
+        file_node = item.find(xpath_with_ns('./jwplayer:source', NS_MAP))
+        
+        video_url = file_node.get('file')
         video = re.search(r'[0-9]+', video_url)
         video_id = video.group(0)
-
-        # Todo: add medium format
-        video_url = video_url.replace(video_id, 'large/' + video_id)
-        if "vimeo" in video_url:
-            video_url = video_url.replace('large/' + video_id, video_id)
-            video_url = video_url.replace('http://stream.gamekings.tv/', '')
+        
+        # Todo: Add medium format
 
         return {
             'id': video_id,

From 641eb10d34629c177445dd38e9d3f1f73c4cb92f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Feb 2015 17:45:38 +0200
Subject: [PATCH 809/974] Use _family_friendly_search for determining age_limit

---
 youtube_dl/extractor/aparat.py   | 2 ++
 youtube_dl/extractor/goshgay.py  | 4 +---
 youtube_dl/extractor/izlesene.py | 5 +----
 youtube_dl/extractor/teamcoco.py | 7 +++++--
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 5d70758a3..63429780e 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -20,6 +20,7 @@ class AparatIE(InfoExtractor):
             'id': 'wP8On',
             'ext': 'mp4',
             'title': 'تیم گلکسی 11 - زومیت',
+            'age_limit': 0,
         },
         # 'skip': 'Extremely unreliable',
     }
@@ -55,4 +56,5 @@ class AparatIE(InfoExtractor):
             'url': video_url,
             'ext': 'mp4',
             'thumbnail': thumbnail,
+            'age_limit': self._family_friendly_search(webpage),
         }
diff --git a/youtube_dl/extractor/goshgay.py b/youtube_dl/extractor/goshgay.py
index b116d251d..1d9166455 100644
--- a/youtube_dl/extractor/goshgay.py
+++ b/youtube_dl/extractor/goshgay.py
@@ -34,8 +34,6 @@ class GoshgayIE(InfoExtractor):
         duration = parse_duration(self._html_search_regex(
             r'<span class="duration">\s*-?\s*(.*?)</span>',
             webpage, 'duration', fatal=False))
-        family_friendly = self._html_search_meta(
-            'isFamilyFriendly', webpage, default='false')
 
         flashvars = compat_parse_qs(self._html_search_regex(
             r'<embed.+?id="flash-player-embed".+?flashvars="([^"]+)"',
@@ -49,5 +47,5 @@ class GoshgayIE(InfoExtractor):
             'title': title,
             'thumbnail': thumbnail,
             'duration': duration,
-            'age_limit': 0 if family_friendly == 'true' else 18,
+            'age_limit': self._family_friendly_search(webpage),
         }
diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index d16d483ee..99a1361f8 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -80,9 +80,6 @@ class IzleseneIE(InfoExtractor):
             r'comment_count\s*=\s*\'([^\']+)\';',
             webpage, 'comment_count', fatal=False)
 
-        family_friendly = self._html_search_meta(
-            'isFamilyFriendly', webpage, 'age limit', fatal=False)
-
         content_url = self._html_search_meta(
             'contentURL', webpage, 'content URL', fatal=False)
         ext = determine_ext(content_url, 'mp4')
@@ -120,6 +117,6 @@ class IzleseneIE(InfoExtractor):
             'duration': duration,
             'view_count': int_or_none(view_count),
             'comment_count': int_or_none(comment_count),
-            'age_limit': 18 if family_friendly == 'False' else 0,
+            'age_limit': self._family_friendly_search(webpage),
             'formats': formats,
         }
diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 18a823719..e85d452a3 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -15,7 +15,8 @@ class TeamcocoIE(InfoExtractor):
                 'id': '80187',
                 'ext': 'mp4',
                 'title': 'Conan Becomes A Mary Kay Beauty Consultant',
-                'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.'
+                'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.',
+                'age_limit': 0,
             }
         }, {
             'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
@@ -24,7 +25,8 @@ class TeamcocoIE(InfoExtractor):
                 'id': '19705',
                 'ext': 'mp4',
                 "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
-                "title": "Louis C.K. Interview Pt. 1 11/3/11"
+                "title": "Louis C.K. Interview Pt. 1 11/3/11",
+                'age_limit': 0,
             }
         }
     ]
@@ -83,4 +85,5 @@ class TeamcocoIE(InfoExtractor):
             'title': self._og_search_title(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
+            'age_limit': self._family_friendly_search(webpage),
         }

From c36b09a5026172c1ca452038fffccd68b14c528c Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 16:46:13 +0100
Subject: [PATCH 810/974] [Gamekings] Use thumbnail in return statement

---
 youtube_dl/extractor/gamekings.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 3c1f7f195..1821f26d8 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -66,4 +66,5 @@ class GamekingsIE(InfoExtractor):
             'url': video_url,
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
+            'thumbnail': image
         }

From d87ec897e99d503e30fb677b0b4acf7256fa6143 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 23:03:12 +0600
Subject: [PATCH 811/974] [gamekings] Improve extraction

---
 youtube_dl/extractor/gamekings.py | 64 ++++++++++++++-----------------
 1 file changed, 29 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 1821f26d8..929779f60 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -1,70 +1,64 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     xpath_text,
-    xpath_with_ns
- )
+    xpath_with_ns,
+)
 
 
 class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
-    _TESTS = [
-        {
+    _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<id>[^/]+)'
+    _TESTS = [{
         'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
         # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
         'info_dict': {
-            'id': '20130811',
+            'id': 'phoenix-wright-ace-attorney-dual-destinies-review',
             'ext': 'mp4',
             'title': 'Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review',
             'description': 'md5:36fd701e57e8c15ac8682a2374c99731',
-            }
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
-        {
+    }, {
+        # vimeo video
         'url': 'http://www.gamekings.tv/videos/the-legend-of-zelda-majoras-mask/',
+        'md5': '12bf04dfd238e70058046937657ea68d',
         'info_dict': {
-            'id': '118933752',
+            'id': 'the-legend-of-zelda-majoras-mask',
             'ext': 'mp4',
             'title': 'The Legend of Zelda: Majora’s Mask',
-            'description': 'md5:9917825fe0e9f4057601fe1e38860de3'
-            }
-        }
-    ]
+            'description': 'md5:9917825fe0e9f4057601fe1e38860de3',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }]
 
     def _real_extract(self, url):
+        video_id = self._match_id(url)
 
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
-        webpage = self._download_webpage(url, name)
+        webpage = self._download_webpage(url, video_id)
 
-        playlist_id = re.search(r'(?:gogoVideo)\(\d+,"?(?P<playlist_id>.*)"', webpage, re.MULTILINE).group('playlist_id')
-        playlist_url = 'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=' + playlist_id
-        playlist_rss = self._download_xml(playlist_url, playlist_id)
-        
+        playlist_id = self._search_regex(
+            r'gogoVideo\(\s*\d+\s*,\s*"([^"]+)', webpage, 'playlist id')
+
+        playlist = self._download_xml(
+            'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=%s' % playlist_id,
+            video_id)
 
         NS_MAP = {
             'jwplayer': 'http://rss.jwpcdn.com/'
-         }
+        }
 
-        item = playlist_rss.find('./channel/item')
-        
-        image = xpath_text(item, xpath_with_ns('./jwplayer:image', NS_MAP), 'image')
-        file_node = item.find(xpath_with_ns('./jwplayer:source', NS_MAP))
-        
-        video_url = file_node.get('file')
-        video = re.search(r'[0-9]+', video_url)
-        video_id = video.group(0)
-        
-        # Todo: Add medium format
+        item = playlist.find('./channel/item')
+
+        thumbnail = xpath_text(item, xpath_with_ns('./jwplayer:image', NS_MAP), 'thumbnail')
+        video_url = item.find(xpath_with_ns('./jwplayer:source', NS_MAP)).get('file')
 
         return {
             'id': video_id,
-            'ext': 'mp4',
             'url': video_url,
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
-            'thumbnail': image
+            'thumbnail': thumbnail,
         }

From 41c23b0da58ba36805a67f4736890475ab7abedf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 23:12:59 +0600
Subject: [PATCH 812/974] [gamekings] Support videos from news pages

---
 youtube_dl/extractor/gamekings.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 929779f60..027f55eb2 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<id>[^/]+)'
+    _VALID_URL = r'http://www\.gamekings\.tv/(?:videos|nieuws)/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
@@ -32,6 +32,9 @@ class GamekingsIE(InfoExtractor):
             'description': 'md5:9917825fe0e9f4057601fe1e38860de3',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.gamekings.tv/nieuws/gamekings-extra-shelly-en-david-bereiden-zich-voor-op-de-livestream/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 12931e1c6eb85adeffa19dcbd3b911335a5ab032 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 23:33:29 +0600
Subject: [PATCH 813/974] Credit @robin007bond for tweakers (#4881) and
 gamekings fixes (#4901)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 85ca81a27..3d6985ab6 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -109,3 +109,4 @@ David Luhmer
 Shaya Goldberg
 Paul Hartmann
 Frans de Jonge
+Robin de Rooij

From 9ddb6925bfc53e0f31f7d114490259e293e91edd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:07:43 +0600
Subject: [PATCH 814/974] [YoutubeDL] Allow filtering by string properties
 (#4906)

---
 youtube_dl/YoutubeDL.py | 41 +++++++++++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 633e3d8a1..0934911ec 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -832,21 +832,38 @@ class YoutubeDL(object):
             \]$
             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
         m = operator_rex.search(format_spec)
+        if m:
+            try:
+                comparison_value = int(m.group('value'))
+            except ValueError:
+                comparison_value = parse_filesize(m.group('value'))
+                if comparison_value is None:
+                    comparison_value = parse_filesize(m.group('value') + 'B')
+                if comparison_value is None:
+                    raise ValueError(
+                        'Invalid value %r in format specification %r' % (
+                            m.group('value'), format_spec))
+            op = OPERATORS[m.group('op')]
+
+        if not m:
+            STR_OPERATORS = {
+                '=': operator.eq,
+                '!=': operator.ne,
+            }
+            str_operator_rex = re.compile(r'''(?x)\s*\[
+                \s*(?P<key>ext|acodec|vcodec|container|protocol)
+                \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
+                \s*(?P<value>[a-zA-Z0-9_-]+)
+                \s*\]$
+                ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
+            m = str_operator_rex.search(format_spec)
+            if m:
+                comparison_value = m.group('value')
+                op = STR_OPERATORS[m.group('op')]
+
         if not m:
             raise ValueError('Invalid format specification %r' % format_spec)
 
-        try:
-            comparison_value = int(m.group('value'))
-        except ValueError:
-            comparison_value = parse_filesize(m.group('value'))
-            if comparison_value is None:
-                comparison_value = parse_filesize(m.group('value') + 'B')
-            if comparison_value is None:
-                raise ValueError(
-                    'Invalid value %r in format specification %r' % (
-                        m.group('value'), format_spec))
-        op = OPERATORS[m.group('op')]
-
         def _filter(f):
             actual_value = f.get(m.group('key'))
             if actual_value is None:

From 2ec19e955870e82bd91949b0996dfd9acdde0232 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:09:45 +0600
Subject: [PATCH 815/974] [YoutubeDL] Allow filtering by audio sampling rate

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0934911ec..c7108dbc1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -826,7 +826,7 @@ class YoutubeDL(object):
             '!=': operator.ne,
         }
         operator_rex = re.compile(r'''(?x)\s*\[
-            (?P<key>width|height|tbr|abr|vbr|filesize|fps)
+            (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
             \]$

From 6128bf07a91b941b9241c5ea0d611df9e69d48b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:27:27 +0600
Subject: [PATCH 816/974] [options] Update help on string comparisons

---
 youtube_dl/options.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index d5e575964..6a795e750 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -298,7 +298,9 @@ def parseOpts(overrideArguments=None):
             ' brackets, as in -f "best[height=720]"'
             ' (or -f "[filesize>10M]"). '
             ' This works for filesize, height, width, tbr, abr, vbr, and fps'
-            ' and the comparisons <, <=, >, >=, =, != .'
+            ' and the comparisons <, <=, >, >=, =, !='
+            ' and for ext, acodec, vcodec, container and protocol'
+            ' and the comparisons =, != .'
             ' Formats for which the value is not known are excluded unless you'
             ' put a question mark (?) after the operator.'
             ' You can combine format filters, so  '

From b29231c0401b7b70337bb52b0c20bc92aa3b704e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 8 Feb 2015 20:28:38 +0100
Subject: [PATCH 817/974] release 2015.02.08

---
 README.md              | 4 +++-
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 06dea400d..3eafad7c7 100644
--- a/README.md
+++ b/README.md
@@ -294,7 +294,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      -f "[filesize>10M]").  This works for
                                      filesize, height, width, tbr, abr, vbr, and
                                      fps and the comparisons <, <=, >, >=, =, !=
-                                     . Formats for which the value is not known
+                                     and for ext, acodec, vcodec, container and
+                                     protocol and the comparisons =, != .
+                                     Formats for which the value is not known
                                      are excluded unless you put a question mark
                                      (?) after the operator. You can combine
                                      format filters, so  -f "[height <=?
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2d8f9c316..84a132684 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -14,6 +14,7 @@
  - **AddAnime**
  - **AdobeTV**
  - **AdultSwim**
+ - **Aftenposten**
  - **Aftonbladet**
  - **AlJazeera**
  - **Allocine**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1091ae61b..b0ddff6e2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.06'
+__version__ = '2015.02.08'

From 64f9baa084c60b8bfd8e33179fd6caa50084a642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:35:16 +0600
Subject: [PATCH 818/974] [options] Mention asr as possible filter

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6a795e750..048d9992c 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -297,9 +297,9 @@ def parseOpts(overrideArguments=None):
             ' You can filter the video results by putting a condition in'
             ' brackets, as in -f "best[height=720]"'
             ' (or -f "[filesize>10M]"). '
-            ' This works for filesize, height, width, tbr, abr, vbr, and fps'
+            ' This works for filesize, height, width, tbr, abr, vbr, asr, and fps'
             ' and the comparisons <, <=, >, >=, =, !='
-            ' and for ext, acodec, vcodec, container and protocol'
+            ' and for ext, acodec, vcodec, container, and protocol'
             ' and the comparisons =, != .'
             ' Formats for which the value is not known are excluded unless you'
             ' put a question mark (?) after the operator.'

From bdb186f3b074f1f7ac0f154be419ca04241add6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20M=C3=BCller?= <mmu@grummel.net>
Date: Sun, 8 Feb 2015 21:55:39 +0100
Subject: [PATCH 819/974] fix rtlnow for newer series like "Der Bachelor"
 season 5

---
 youtube_dl/extractor/rtlnow.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index 285c3c4be..e235f3c57 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -134,9 +134,18 @@ class RTLnowIE(InfoExtractor):
                     'player_url': video_page_url + 'includes/vodplayer.swf',
                 }
             else:
-                fmt = {
-                    'url': filename.text,
-                }
+                mobj = re.search(r'.*/(?P<hoster>[^/]+)/videos/(?P<play_path>.+)\.f4m', filename.text)
+                if mobj:
+                    fmt = {
+                        'url': 'rtmpe://fmspay-fra2.rtl.de/' + mobj.group('hoster'),
+                        'play_path': 'mp4:' + mobj.group('play_path'),
+                        'page_url': url,
+                        'player_url': video_page_url + 'includes/vodplayer.swf',
+                    }
+                else:
+                    fmt = {
+                        'url': filename.text,
+                    }
             fmt.update({
                 'width': int_or_none(filename.get('width')),
                 'height': int_or_none(filename.get('height')),

From dd8930684ea4069a1c4e28b3298b0d328145d00f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:28:16 +0100
Subject: [PATCH 820/974] release 2015.02.09

---
 README.md             | 28 ++++++++++++++--------------
 youtube_dl/version.py |  2 +-
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index 3eafad7c7..23295df4b 100644
--- a/README.md
+++ b/README.md
@@ -292,20 +292,20 @@ which means you can modify it, redistribute it or use it however you like.
                                      video results by putting a condition in
                                      brackets, as in -f "best[height=720]" (or
                                      -f "[filesize>10M]").  This works for
-                                     filesize, height, width, tbr, abr, vbr, and
-                                     fps and the comparisons <, <=, >, >=, =, !=
-                                     and for ext, acodec, vcodec, container and
-                                     protocol and the comparisons =, != .
-                                     Formats for which the value is not known
-                                     are excluded unless you put a question mark
-                                     (?) after the operator. You can combine
-                                     format filters, so  -f "[height <=?
-                                     720][tbr>500]" selects up to 720p videos
-                                     (or videos where the height is not known)
-                                     with a bitrate of at least 500 KBit/s. By
-                                     default, youtube-dl will pick the best
-                                     quality. Use commas to download multiple
-                                     audio formats, such as -f
+                                     filesize, height, width, tbr, abr, vbr,
+                                     asr, and fps and the comparisons <, <=, >,
+                                     >=, =, != and for ext, acodec, vcodec,
+                                     container, and protocol and the comparisons
+                                     =, != . Formats for which the value is not
+                                     known are excluded unless you put a
+                                     question mark (?) after the operator. You
+                                     can combine format filters, so  -f "[height
+                                     <=? 720][tbr>500]" selects up to 720p
+                                     videos (or videos where the height is not
+                                     known) with a bitrate of at least 500
+                                     KBit/s. By default, youtube-dl will pick
+                                     the best quality. Use commas to download
+                                     multiple audio formats, such as -f
                                      136/137/mp4/bestvideo,140/m4a/bestaudio.
                                      You can merge the video and audio of two
                                      formats into a single file using -f <video-
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b0ddff6e2..b62fcd63a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.08'
+__version__ = '2015.02.09'

From a0f719854463c6f4226e4042dfa80c1b17154e1d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:42:25 +0100
Subject: [PATCH 821/974] [generic] Add support for jwPlayer YouTube videos

This makes nationalarchives.gov.uk work (Fixes #4907, fixes #4876)
---
 youtube_dl/extractor/generic.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index fbbc79a57..8a0f10803 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -524,6 +524,19 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20150126',
             },
             'add_ie': ['Viddler'],
+        },
+        # jwplayer YouTube
+        {
+            'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
+            'info_dict': {
+                'id': 'Mrj4DVp2zeA',
+                'ext': 'mp4',
+                'upload_date': '20150204',
+                'uploader': 'The National Archives UK',
+                'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
+                'uploader_id': 'NationalArchives08',
+                'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
+            },
         }
     ]
 
@@ -1065,6 +1078,8 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'Livestream')
 
         def check_video(vurl):
+            if YoutubeIE.suitable(vurl):
+                return True
             vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath)
             return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
@@ -1082,7 +1097,8 @@ class GenericIE(InfoExtractor):
                     JWPlayerOptions|
                     jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
                 )
-                .*?file\s*:\s*["\'](.*?)["\']''', webpage))
+                .*?
+                ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
         if not found:
             # Broaden the search a little bit
             found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))

From 5269028951d5abd6f20d46c267b0ef610b2becef Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:47:19 +0100
Subject: [PATCH 822/974] [rtlnow] Add test for @mmue's extension (#4908)

---
 youtube_dl/extractor/rtlnow.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index e235f3c57..fd93cc66f 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -91,6 +91,15 @@ class RTLnowIE(InfoExtractor):
             },
         },
         {
+            'url': 'http://rtl-now.rtl.de/der-bachelor/folge-4.php?film_id=188729&player=1&season=5',
+            'info_dict': {
+                'id': '188729',
+                'ext': 'flv',
+                'upload_date': '20150204',
+                'description': 'md5:5e1ce23095e61a79c166d134b683cecc',
+                'title': 'Der Bachelor - Folge 4',
+            }
+        }, {
             'url': 'http://www.n-tvnow.de/deluxe-alles-was-spass-macht/thema-ua-luxushotel-fuer-vierbeiner.php?container_id=153819&player=1&season=0',
             'only_matching': True,
         },

From 196121c51baf2e7b7bb9c657f2f832b2840432de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:49:10 +0100
Subject: [PATCH 823/974] release 2015.02.09.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b62fcd63a..e3f6d1e90 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09'
+__version__ = '2015.02.09.1'

From e98b8e79ead03f33705356b32a08664ef2df2628 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 14:46:10 +0100
Subject: [PATCH 824/974] [generic] Improve SBS detection (Fixes #4899)

---
 youtube_dl/extractor/generic.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8a0f10803..f4500e931 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1047,7 +1047,12 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded sbs.com.au player
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)sbs\.com\.au/ondemand/video/single/.+?)\1',
+            r'''(?x)
+            (?:
+                <meta\s+property="og:video"\s+content=|
+                <iframe[^>]+?src=
+            )
+            (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
             webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'SBS')

From 3255fe7141a1e11d65997da3de436ba3007a8fc3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 14:46:30 +0100
Subject: [PATCH 825/974] release 2015.02.09.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e3f6d1e90..d8911cae6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09.1'
+__version__ = '2015.02.09.2'

From ba617964585a118e30932461f24a6c3627047738 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 9 Feb 2015 15:04:22 +0100
Subject: [PATCH 826/974] [youtube] Don't override format info from the dash
 manifest (fixes #4911)

---
 youtube_dl/extractor/youtube.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e4b26b84f..8b91fff18 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -780,8 +780,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     fo for fo in formats
                     if fo['format_id'] == format_id)
             except StopIteration:
-                f.update(self._formats.get(format_id, {}).items())
-                formats.append(f)
+                full_info = self._formats.get(format_id, {}).copy()
+                full_info.update(f)
+                formats.append(full_info)
             else:
                 existing_format.update(f)
         return formats

From 1309b396d0ff87f0e762f8893b6d0bd2aa96432a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 15:56:46 +0100
Subject: [PATCH 827/974] [svtplay] Add new extractor (Fixes #4914)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/svtplay.py  | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/svtplay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0d7a120bc..55ca0d6e4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -428,6 +428,7 @@ from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
+from .svtplay import SVTPlayIE
 from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py
new file mode 100644
index 000000000..727b3d1fc
--- /dev/null
+++ b/youtube_dl/extractor/svtplay.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+)
+
+
+class SVTPlayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?svtplay\.se/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
+        'md5': '2521cd644e862936cf2e698206e47385',
+        'info_dict': {
+            'id': '3966754',
+            'ext': 'mp4',
+            'title': 'FIFA 14 - E3 2013 Trailer',
+            'duration': 4500,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        info = self._download_json(
+            'http://www.svtplay.se/video/%s?output=json' % video_id, video_id)
+
+        title = info['context']['title']
+        thumbnail = info['context'].get('thumbnailImage')
+
+        video_info = info['video']
+        formats = []
+        for vr in video_info['videoReferences']:
+            vurl = vr['url']
+            if determine_ext(vurl) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    vurl, video_id,
+                    ext='mp4', entry_protocol='m3u8_native',
+                    m3u8_id=vr.get('playerType')))
+            else:
+                formats.append({
+                    'format_id': vr.get('playerType'),
+                    'url': vurl,
+                })
+        self._sort_formats(formats)
+
+        duration = video_info.get('materialLength')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

From 43f244b6d51aefac4683d130d70adf94641bf6d8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 15:57:40 +0100
Subject: [PATCH 828/974] [YoutubeDL] Do not show worst in --list-formats
 output

Nobody wants to know what the worst possible format is. And if they do, they can still provide -f worst.
---
 youtube_dl/YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c7108dbc1..3975ae0bc 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1546,7 +1546,6 @@ class YoutubeDL(object):
             line(f, idlen) for f in formats
             if f.get('preference') is None or f['preference'] >= -1000]
         if len(formats) > 1:
-            formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)'
             formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
 
         header_line = line({

From 2caf182f37562a8dc3857f88a19f6902f6d829e7 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 9 Feb 2015 16:55:31 +0200
Subject: [PATCH 829/974] [trilulilu] Add support for videos without category
 in the URL (Closes #4067)

Also, update the testcase, detect private/country restricted videos and modernize a bit.
---
 youtube_dl/extractor/trilulilu.py | 51 ++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py
index 220a05b7b..185accc4b 100644
--- a/youtube_dl/extractor/trilulilu.py
+++ b/youtube_dl/extractor/trilulilu.py
@@ -1,40 +1,55 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import json
+import re
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class TriluliluIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?trilulilu\.ro/video-[^/]+/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?trilulilu\.ro/(?:video-[^/]+/)?(?P<id>[^/#\?]+)'
     _TEST = {
         'url': 'http://www.trilulilu.ro/video-animatie/big-buck-bunny-1',
+        'md5': 'c1450a00da251e2769b74b9005601cac',
         'info_dict': {
-            'id': 'big-buck-bunny-1',
+            'id': 'ae2899e124140b',
             'ext': 'mp4',
             'title': 'Big Buck Bunny',
             'description': ':) pentru copilul din noi',
         },
-        # Server ignores Range headers (--test)
-        'params': {
-            'skip_download': True
-        }
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
 
+        if re.search(r'Fişierul nu este disponibil pentru vizionare în ţara dumneavoastră', webpage):
+            raise ExtractorError(
+                'This video is not available in your country.', expected=True)
+        elif re.search('Fişierul poate fi accesat doar de către prietenii lui', webpage):
+            raise ExtractorError('This video is private.', expected=True)
+
+        flashvars_str = self._search_regex(
+            r'block_flash_vars\s*=\s*(\{[^\}]+\})', webpage, 'flashvars', fatal=False, default=None)
+
+        if flashvars_str:
+            flashvars = self._parse_json(flashvars_str, display_id)
+        else:
+            raise ExtractorError(
+                'This page does not contain videos', expected=True)
+
+        if flashvars['isMP3'] == 'true':
+            raise ExtractorError(
+                'Audio downloads are currently not supported', expected=True)
+
+        video_id = flashvars['hash']
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        description = self._og_search_description(webpage)
-
-        log_str = self._search_regex(
-            r'block_flash_vars[ ]=[ ]({[^}]+})', webpage, 'log info')
-        log = json.loads(log_str)
+        description = self._og_search_description(webpage, default=None)
 
         format_url = ('http://fs%(server)s.trilulilu.ro/%(hash)s/'
-                      'video-formats2' % log)
+                      'video-formats2' % flashvars)
         format_doc = self._download_xml(
             format_url, video_id,
             note='Downloading formats',
@@ -44,10 +59,10 @@ class TriluliluIE(InfoExtractor):
             'http://fs%(server)s.trilulilu.ro/stream.php?type=video'
             '&source=site&hash=%(hash)s&username=%(userid)s&'
             'key=ministhebest&format=%%s&sig=&exp=' %
-            log)
+            flashvars)
         formats = [
             {
-                'format': fnode.text,
+                'format_id': fnode.text.partition('-')[2],
                 'url': video_url_template % fnode.text,
                 'ext': fnode.text.partition('-')[0]
             }
@@ -56,8 +71,8 @@ class TriluliluIE(InfoExtractor):
         ]
 
         return {
-            '_type': 'video',
             'id': video_id,
+            'display_id': display_id,
             'formats': formats,
             'title': title,
             'description': description,

From f18e3a2fc0b73011c98c39d54576fb835cee4561 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 15:59:19 +0100
Subject: [PATCH 830/974] release 2015.02.09.3

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 84a132684..93572f6a8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -392,6 +392,7 @@
  - **StreamCZ**
  - **StreetVoice**
  - **SunPorno**
+ - **SVTPlay**
  - **SWRMediathek**
  - **Syfy**
  - **SztvHu**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d8911cae6..30447a368 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09.2'
+__version__ = '2015.02.09.3'

From 28f1272870c43700a90af4671b6eae46d0fab6c6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 16:05:01 +0100
Subject: [PATCH 831/974] [svtplay] Correct test case

---
 youtube_dl/extractor/svtplay.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py
index 727b3d1fc..eadb9ccb4 100644
--- a/youtube_dl/extractor/svtplay.py
+++ b/youtube_dl/extractor/svtplay.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
@@ -10,13 +11,13 @@ class SVTPlayIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?svtplay\.se/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
-        'md5': '2521cd644e862936cf2e698206e47385',
+        'md5': 'f4a184968bc9c802a9b41316657aaa80',
         'info_dict': {
-            'id': '3966754',
+            'id': '2609989',
             'ext': 'mp4',
-            'title': 'FIFA 14 - E3 2013 Trailer',
+            'title': 'SM veckan vinter, Örebro - Rally, final',
             'duration': 4500,
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
         },
     }
 

From 99c2398bc641c478dbea85c013081d883479827e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 19:08:51 +0100
Subject: [PATCH 832/974] [bandcamp] Use our API to get more stable error
 messages (#1194)

---
 youtube_dl/extractor/bandcamp.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index aea0263d6..27d8652f6 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -72,26 +72,29 @@ class BandcampIE(InfoExtractor):
 
         download_link = m_download.group(1)
         video_id = self._search_regex(
-            r'var TralbumData = {.*?id: (?P<id>\d+),?$',
-            webpage, 'video id', flags=re.MULTILINE | re.DOTALL)
+            r'(?ms)var TralbumData = {.*?id: (?P<id>\d+),?$',
+            webpage, 'video id')
 
         download_webpage = self._download_webpage(download_link, video_id, 'Downloading free downloads page')
         # We get the dictionary of the track from some javascript code
-        info = re.search(r'items: (.*?),$', download_webpage, re.MULTILINE).group(1)
-        info = json.loads(info)[0]
+        all_info = self._parse_json(self._search_regex(
+            r'(?sm)items: (.*?),$', download_webpage, 'items'), video_id)
+        info = All_info[0]
         # We pick mp3-320 for now, until format selection can be easily implemented.
         mp3_info = info['downloads']['mp3-320']
         # If we try to use this url it says the link has expired
         initial_url = mp3_info['url']
-        re_url = r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$'
-        m_url = re.match(re_url, initial_url)
+        m_url = re.match(
+            r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$',
+            initial_url)
         # We build the url we will use to get the final track url
         # This url is build in Bandcamp in the script download_bunde_*.js
         request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), video_id, m_url.group('ts'))
         final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url')
         # If we could correctly generate the .rand field the url would be
         # in the "download_url" key
-        final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1)
+        final_url = self._search_regex(
+            r'"retry_url":"(.*?)"', final_url_webpage, 'final video URL')
 
         return {
             'id': video_id,

From 3a5bcd03263059bd7b905d4586c5f23f7f120974 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:13:57 +0100
Subject: [PATCH 833/974] [extractor/common] Wrap extractor errors (Fixes
 #1194)

For now, we just wrap some common errors. More may follow. We do not want to catch actual programming errors in the extractors, such as 1 // 0.
---
 youtube_dl/extractor/common.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2f5ba7aee..eee936a6f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -264,8 +264,15 @@ class InfoExtractor(object):
 
     def extract(self, url):
         """Extracts URL information and returns it in list of dicts."""
-        self.initialize()
-        return self._real_extract(url)
+        try:
+            self.initialize()
+            return self._real_extract(url)
+        except ExtractorError:
+            raise
+        except compat_http_client.IncompleteRead as e:
+            raise ExtractorError('A network error has occured.', cause=e, expected=True)
+        except (KeyError,) as e:
+            raise ExtractorError('An extractor error has occured.', cause=e)
 
     def set_downloader(self, downloader):
         """Sets the downloader for this IE."""

From 34814eb66e948b5d73420e435f3aea7112b243d5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:19:52 +0100
Subject: [PATCH 834/974] release 2015.02.10

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 30447a368..2414627da 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09.3'
+__version__ = '2015.02.10'

From 753fad4adc32b57a0d18518fddd06d2411d09635 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:34:01 +0100
Subject: [PATCH 835/974] [commonmistakes] Correct logic error

---
 youtube_dl/extractor/commonmistakes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py
index 75c06903f..dbbf27a74 100644
--- a/youtube_dl/extractor/commonmistakes.py
+++ b/youtube_dl/extractor/commonmistakes.py
@@ -24,6 +24,6 @@ class CommonMistakesIE(InfoExtractor):
             'That doesn\'t make any sense. '
             'Simply remove the parameter in your command or configuration.'
         ) % url
-        if self._downloader.params.get('verbose'):
+        if not self._downloader.params.get('verbose'):
             msg += ' Add -v to the command line to see what arguments and configuration youtube-dl got.'
         raise ExtractorError(msg, expected=True)

From c06a9fa34f41edf090f4beb385212e44a6e1bb08 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:36:38 +0100
Subject: [PATCH 836/974] Use snake_case instead of camelCase

---
 youtube_dl/options.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 048d9992c..9d9195b60 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -734,22 +734,22 @@ def parseOpts(overrideArguments=None):
         if opts.verbose:
             write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
     else:
-        commandLineConf = sys.argv[1:]
-        if '--ignore-config' in commandLineConf:
-            systemConf = []
-            userConf = []
+        command_line_conf = sys.argv[1:]
+        if '--ignore-config' in command_line_conf:
+            system_conf = []
+            user_conf = []
         else:
-            systemConf = _readOptions('/etc/youtube-dl.conf')
-            if '--ignore-config' in systemConf:
-                userConf = []
+            system_conf = _readOptions('/etc/youtube-dl.conf')
+            if '--ignore-config' in system_conf:
+                user_conf = []
             else:
-                userConf = _readUserConf()
-        argv = systemConf + userConf + commandLineConf
+                user_conf = _readUserConf()
+        argv = system_conf + user_conf + command_line_conf
 
         opts, args = parser.parse_args(argv)
         if opts.verbose:
-            write_string('[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
-            write_string('[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
-            write_string('[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
+            write_string('[debug] System config: ' + repr(_hide_login_info(system_conf)) + '\n')
+            write_string('[debug] User config: ' + repr(_hide_login_info(user_conf)) + '\n')
+            write_string('[debug] Command-line args: ' + repr(_hide_login_info(command_line_conf)) + '\n')
 
     return parser, opts, args

From 834bf069d20745263586284c98196b5b0605b916 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:37:14 +0100
Subject: [PATCH 837/974] [bandcamp] Correct variable name

---
 youtube_dl/extractor/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 27d8652f6..490cc961a 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -79,7 +79,7 @@ class BandcampIE(InfoExtractor):
         # We get the dictionary of the track from some javascript code
         all_info = self._parse_json(self._search_regex(
             r'(?sm)items: (.*?),$', download_webpage, 'items'), video_id)
-        info = All_info[0]
+        info = all_info[0]
         # We pick mp3-320 for now, until format selection can be easily implemented.
         mp3_info = info['downloads']['mp3-320']
         # If we try to use this url it says the link has expired

From c73fae1e2e4421df664aefd1d14a72596caf9e2f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:39:43 +0100
Subject: [PATCH 838/974] [commonmistakes] Detect BOMs at the beginning of URLs

Reported at https://bugzilla.redhat.com/show_bug.cgi?id=1093517 .
---
 youtube_dl/extractor/__init__.py       |  2 +-
 youtube_dl/extractor/commonmistakes.py | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 55ca0d6e4..fb1e7f325 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -74,7 +74,7 @@ from .collegehumor import CollegeHumorIE
 from .collegerama import CollegeRamaIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
-from .commonmistakes import CommonMistakesIE
+from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py
index dbbf27a74..2f86e2381 100644
--- a/youtube_dl/extractor/commonmistakes.py
+++ b/youtube_dl/extractor/commonmistakes.py
@@ -27,3 +27,20 @@ class CommonMistakesIE(InfoExtractor):
         if not self._downloader.params.get('verbose'):
             msg += ' Add -v to the command line to see what arguments and configuration youtube-dl got.'
         raise ExtractorError(msg, expected=True)
+
+
+class UnicodeBOMIE(InfoExtractor):
+        IE_DESC = False
+        _VALID_URL = r'(?P<bom>\ufeff)(?P<id>.*)$'
+
+        _TESTS = [{
+            'url': '\ufeffhttp://www.youtube.com/watch?v=BaW_jenozKc',
+            'only_matching': True,
+        }]
+
+        def _real_extract(self, url):
+            real_url = self._match_id(url)
+            self.report_warning(
+                'Your URL starts with a Byte Order Mark (BOM). '
+                'Removing the BOM and looking for "%s" ...' % real_url)
+            return self.url_result(real_url)

From 88296505131f4b91ff91eaa0af34318664d892c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:46:09 +0100
Subject: [PATCH 839/974] release 2015.02.10.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2414627da..7d144dadc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10'
+__version__ = '2015.02.10.1'

From 347de4931cb3e496fb7b1dfb0314c213f44cce6b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 03:32:21 +0100
Subject: [PATCH 840/974] [YoutubeDL] Add generic video filtering (Fixes #4916)

This functionality is intended to eventually encompass the current format filtering.
---
 test/test_utils.py      | 32 +++++++++++++++++
 youtube_dl/YoutubeDL.py | 14 +++++++-
 youtube_dl/__init__.py  |  7 +++-
 youtube_dl/options.py   | 19 ++++++++++
 youtube_dl/utils.py     | 77 +++++++++++++++++++++++++++++++++++++++++
 5 files changed, 147 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 80c765bc4..1c29d0889 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -53,6 +53,7 @@ from youtube_dl.utils import (
     version_tuple,
     xpath_with_ns,
     render_table,
+    match_str,
 )
 
 
@@ -459,6 +460,37 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             '123  4\n'
             '9999 51')
 
+    def test_match_str(self):
+        self.assertRaises(ValueError, match_str, 'xy>foobar', {})
+        self.assertFalse(match_str('xy', {'x': 1200}))
+        self.assertTrue(match_str('!xy', {'x': 1200}))
+        self.assertTrue(match_str('x', {'x': 1200}))
+        self.assertFalse(match_str('!x', {'x': 1200}))
+        self.assertTrue(match_str('x', {'x': 0}))
+        self.assertFalse(match_str('x>0', {'x': 0}))
+        self.assertFalse(match_str('x>0', {}))
+        self.assertTrue(match_str('x>?0', {}))
+        self.assertTrue(match_str('x>1K', {'x': 1200}))
+        self.assertFalse(match_str('x>2K', {'x': 1200}))
+        self.assertTrue(match_str('x>=1200 & x < 1300', {'x': 1200}))
+        self.assertFalse(match_str('x>=1100 & x < 1200', {'x': 1200}))
+        self.assertFalse(match_str('y=a212', {'y': 'foobar42'}))
+        self.assertTrue(match_str('y=foobar42', {'y': 'foobar42'}))
+        self.assertFalse(match_str('y!=foobar42', {'y': 'foobar42'}))
+        self.assertTrue(match_str('y!=foobar2', {'y': 'foobar42'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 90, 'description': 'foo'}))
+        self.assertTrue(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'description': 'foo'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'dislike_count': 60, 'description': 'foo'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'dislike_count': 10}))
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3975ae0bc..dda222fee 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -228,6 +228,11 @@ class YoutubeDL(object):
     external_downloader:  Executable of the external downloader to call.
     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
+    match_filter:      A function that gets called with the info_dict of
+                       every video.
+                       If it returns a message, the video is ignored.
+                       If it returns None, the video is downloaded.
+                       match_filter_func in utils.py is one example for this.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -583,9 +588,16 @@ class YoutubeDL(object):
             if max_views is not None and view_count > max_views:
                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
-            return 'Skipping "%s" because it is age restricted' % title
+            return 'Skipping "%s" because it is age restricted' % video_title
         if self.in_download_archive(info_dict):
             return '%s has already been recorded in archive' % video_title
+
+        match_filter = self.params.get('match_filter')
+        if match_filter is not None:
+            ret = match_filter(info_dict)
+            if ret is not None:
+                return ret
+
         return None
 
     @staticmethod
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index e90679ff9..eefca0fe4 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -23,9 +23,10 @@ from .compat import (
 )
 from .utils import (
     DateRange,
-    DEFAULT_OUTTMPL,
     decodeOption,
+    DEFAULT_OUTTMPL,
     DownloadError,
+    match_filter_func,
     MaxDownloadsReached,
     preferredencoding,
     read_batch_urls,
@@ -247,6 +248,9 @@ def _real_main(argv=None):
             xattr  # Confuse flake8
         except ImportError:
             parser.error('setting filesize xattr requested but python-xattr is not available')
+    match_filter = (
+        None if opts.match_filter is None
+        else match_filter_func(opts.match_filter))
 
     ydl_opts = {
         'usenetrc': opts.usenetrc,
@@ -344,6 +348,7 @@ def _real_main(argv=None):
         'list_thumbnails': opts.list_thumbnails,
         'playlist_items': opts.playlist_items,
         'xattr_set_filesize': opts.xattr_set_filesize,
+        'match_filter': match_filter,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 9d9195b60..f64aa5b85 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -244,6 +244,25 @@ def parseOpts(overrideArguments=None):
         '--max-views',
         metavar='COUNT', dest='max_views', default=None, type=int,
         help='Do not download any videos with more than COUNT views')
+    selection.add_option(
+        '--match-filter',
+        metavar='FILTER', dest='match_filter', default=None,
+        help=(
+            '(Experimental) Generic video filter. '
+            'Specify any key (see help for -o for a list of available keys) to'
+            ' match if the key is present, '
+            '!key to check if the key is not present,'
+            'key > NUMBER (like "comment_count > 12", also works with '
+            '>=, <, <=, !=, =) to compare against a number, and '
+            '& to require multiple matches. '
+            'Values which are not known are excluded unless you'
+            ' put a question mark (?) after the operator.'
+            'For example, to only match videos that have been liked more than '
+            '100 times and disliked less than 50 times (or the dislike '
+            'functionality is not available at the given service), but who '
+            'also have a description, use  --match-filter '
+            '"like_count > 100 & dislike_count <? 50 & description" .'
+        ))
     selection.add_option(
         '--no-playlist',
         action='store_true', dest='noplaylist', default=False,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8f5463f1c..03566d223 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -17,6 +17,7 @@ import io
 import json
 import locale
 import math
+import operator
 import os
 import pipes
 import platform
@@ -1678,3 +1679,79 @@ def render_table(header_row, data):
     max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
     format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
     return '\n'.join(format_str % tuple(row) for row in table)
+
+
+def _match_one(filter_part, dct):
+    COMPARISON_OPERATORS = {
+        '<': operator.lt,
+        '<=': operator.le,
+        '>': operator.gt,
+        '>=': operator.ge,
+        '=': operator.eq,
+        '!=': operator.ne,
+    }
+    operator_rex = re.compile(r'''(?x)\s*
+        (?P<key>[a-z_]+)
+        \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
+        (?:
+            (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
+            (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
+        )
+        \s*$
+        ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
+    m = operator_rex.search(filter_part)
+    if m:
+        op = COMPARISON_OPERATORS[m.group('op')]
+        if m.group('strval') is not None:
+            if m.group('op') not in ('=', '!='):
+                raise ValueError(
+                    'Operator %s does not support string values!' % m.group('op'))
+            comparison_value = m.group('strval')
+        else:
+            try:
+                comparison_value = int(m.group('intval'))
+            except ValueError:
+                comparison_value = parse_filesize(m.group('intval'))
+                if comparison_value is None:
+                    comparison_value = parse_filesize(m.group('intval') + 'B')
+                if comparison_value is None:
+                    raise ValueError(
+                        'Invalid integer value %r in filter part %r' % (
+                            m.group('intval'), filter_part))
+        actual_value = dct.get(m.group('key'))
+        if actual_value is None:
+            return m.group('none_inclusive')
+        return op(actual_value, comparison_value)
+
+    UNARY_OPERATORS = {
+        '': lambda v: v is not None,
+        '!': lambda v: v is None,
+    }
+    operator_rex = re.compile(r'''(?x)\s*
+        (?P<op>%s)\s*(?P<key>[a-z_]+)
+        \s*$
+        ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
+    m = operator_rex.search(filter_part)
+    if m:
+        op = UNARY_OPERATORS[m.group('op')]
+        actual_value = dct.get(m.group('key'))
+        return op(actual_value)
+
+    raise ValueError('Invalid filter part %r' % filter_part)
+
+
+def match_str(filter_str, dct):
+    """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
+
+    return all(
+        _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
+
+
+def match_filter_func(filter_str):
+    def _match_func(info_dict):
+        if match_str(filter_str, info_dict):
+            return None
+        else:
+            video_title = info_dict.get('title', info_dict.get('id', 'video'))
+            return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
+    return _match_func

From 845734773d334b1a338129dad7c656c12c8fbf6f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 03:32:55 +0100
Subject: [PATCH 841/974] release 2015.02.10.2

---
 README.md             | 17 +++++++++++++++++
 youtube_dl/version.py |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 23295df4b..7aad3007f 100644
--- a/README.md
+++ b/README.md
@@ -119,6 +119,23 @@ which means you can modify it, redistribute it or use it however you like.
                                      COUNT views
     --max-views COUNT                Do not download any videos with more than
                                      COUNT views
+    --match-filter FILTER            (Experimental) Generic video filter.
+                                     Specify any key (see help for -o for a list
+                                     of available keys) to match if the key is
+                                     present, !key to check if the key is not
+                                     present,key > NUMBER (like "comment_count >
+                                     12", also works with >=, <, <=, !=, =) to
+                                     compare against a number, and & to require
+                                     multiple matches. Values which are not
+                                     known are excluded unless you put a
+                                     question mark (?) after the operator.For
+                                     example, to only match videos that have
+                                     been liked more than 100 times and disliked
+                                     less than 50 times (or the dislike
+                                     functionality is not available at the given
+                                     service), but who also have a description,
+                                     use  --match-filter "like_count > 100 &
+                                     dislike_count <? 50 & description" .
     --no-playlist                    If the URL refers to a video and a
                                      playlist, download only the video.
     --age-limit YEARS                download only videos suitable for the given
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7d144dadc..9ab5b49b8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.1'
+__version__ = '2015.02.10.2'

From f7a211dcc8d86fe5a4d25f4d209ffb3efc05bdfc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 03:41:31 +0100
Subject: [PATCH 842/974] [pornhd] Fix extraction (fixes #4915)

---
 youtube_dl/extractor/pornhd.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 954dfccb7..dbb2c3bd9 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -46,16 +46,17 @@ class PornHdIE(InfoExtractor):
 
         quality = qualities(['sd', 'hd'])
         sources = json.loads(js_to_json(self._search_regex(
-            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}\);", webpage, 'sources')))
+            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}[;,)]",
+            webpage, 'sources')))
         formats = []
-        for container, s in sources.items():
-            for qname, video_url in s.items():
-                formats.append({
-                    'url': video_url,
-                    'container': container,
-                    'format_id': '%s-%s' % (container, qname),
-                    'quality': quality(qname),
-                })
+        for qname, video_url in sources.items():
+            if not video_url:
+                continue
+            formats.append({
+                'url': video_url,
+                'format_id': qname,
+                'quality': quality(qname),
+            })
         self._sort_formats(formats)
 
         return {

From 7e5db8c930e5f154287f98c6c0da90fffd2ede4f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 04:22:10 +0100
Subject: [PATCH 843/974] [options] Add --no-color

---
 youtube_dl/YoutubeDL.py        | 5 +++--
 youtube_dl/__init__.py         | 1 +
 youtube_dl/extractor/common.py | 2 +-
 youtube_dl/options.py          | 5 +++++
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dda222fee..e2ec55c90 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -233,6 +233,7 @@ class YoutubeDL(object):
                        If it returns a message, the video is ignored.
                        If it returns None, the video is downloaded.
                        match_filter_func in utils.py is one example for this.
+    no_color:          Do not emit color codes in output.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -490,7 +491,7 @@ class YoutubeDL(object):
         else:
             if self.params.get('no_warnings'):
                 return
-            if self._err_file.isatty() and os.name != 'nt':
+            if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
                 _msg_header = '\033[0;33mWARNING:\033[0m'
             else:
                 _msg_header = 'WARNING:'
@@ -502,7 +503,7 @@ class YoutubeDL(object):
         Do the same as trouble, but prefixes the message with 'ERROR:', colored
         in red if stderr is a tty file.
         '''
-        if self._err_file.isatty() and os.name != 'nt':
+        if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
             _msg_header = '\033[0;31mERROR:\033[0m'
         else:
             _msg_header = 'ERROR:'
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index eefca0fe4..ed22f169f 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -349,6 +349,7 @@ def _real_main(argv=None):
         'playlist_items': opts.playlist_items,
         'xattr_set_filesize': opts.xattr_set_filesize,
         'match_filter': match_filter,
+        'no_color': opts.no_color,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index eee936a6f..382e846c5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -514,7 +514,7 @@ class InfoExtractor(object):
                 if mobj:
                     break
 
-        if os.name != 'nt' and sys.stderr.isatty():
+        if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():
             _name = '\033[0;34m%s\033[0m' % name
         else:
             _name = name
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f64aa5b85..0219e6ba4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -165,6 +165,11 @@ def parseOpts(overrideArguments=None):
         action='store_const', dest='extract_flat', const='in_playlist',
         default=False,
         help='Do not extract the videos of a playlist, only list them.')
+    general.add_option(
+        '--no-color', '--no-colors',
+        action='store_true', dest='no_color',
+        default=False,
+        help='Do not emit color codes in output.')
 
     network = optparse.OptionGroup(parser, 'Network Options')
     network.add_option(

From 2f543a2142e05bac988ccd6566e0c9fc5cadca90 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 04:52:33 +0100
Subject: [PATCH 844/974] [options] Add alias --dump-header for --print-traffic

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0219e6ba4..873432bee 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -557,7 +557,7 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='youtube_print_sig_code', default=False,
         help=optparse.SUPPRESS_HELP)
     verbosity.add_option(
-        '--print-traffic',
+        '--print-traffic', '--dump-headers',
         dest='debug_printtraffic', action='store_true', default=False,
         help='Display sent and read HTTP traffic')
     verbosity.add_option(

From bf8f082a90f4396a0a70464b55bf5c11fdb36c47 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 04:53:21 +0100
Subject: [PATCH 845/974] [vimeo:album] Add support for album passwords (Fixes
 #4917)

---
 youtube_dl/extractor/vimeo.py | 59 ++++++++++++++++++++++++++++++-----
 1 file changed, 52 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 1bb473514..303e81447 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -188,9 +188,9 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
         password_request = compat_urllib_request.Request(pass_url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Cookie', 'xsrft=%s' % token)
-        self._download_webpage(password_request, video_id,
-                               'Verifying the password',
-                               'Wrong password')
+        return self._download_webpage(
+            password_request, video_id,
+            'Verifying the password', 'Wrong password')
 
     def _verify_player_video_password(self, url, video_id):
         password = self._downloader.params.get('videopassword', None)
@@ -266,7 +266,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
             if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
                 raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
 
-            if re.search('<form[^>]+?id="pw_form"', webpage) is not None:
+            if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
                 self._verify_video_password(url, video_id, webpage)
                 return self._real_extract(url)
             else:
@@ -412,12 +412,47 @@ class VimeoChannelIE(InfoExtractor):
     def _extract_list_title(self, webpage):
         return self._html_search_regex(self._TITLE_RE, webpage, 'list title')
 
+    def _login_list_password(self, page_url, list_id, webpage):
+        login_form = self._search_regex(
+            r'(?s)<form[^>]+?id="pw_form"(.*?)</form>',
+            webpage, 'login form', default=None)
+        if not login_form:
+            return webpage
+
+        password = self._downloader.params.get('videopassword', None)
+        if password is None:
+            raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
+        fields = dict(re.findall(r'''(?x)<input\s+
+            type="hidden"\s+
+            name="([^"]+)"\s+
+            value="([^"]*)"
+            ''', login_form))
+        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
+        fields['token'] = token
+        fields['password'] = password
+        post = compat_urllib_parse.urlencode(fields)
+        password_path = self._search_regex(
+            r'action="([^"]+)"', login_form, 'password URL')
+        password_url = compat_urlparse.urljoin(page_url, password_path)
+        password_request = compat_urllib_request.Request(password_url, post)
+        password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
+        self._set_cookie('vimeo.com', 'xsrft', token)
+
+        return self._download_webpage(
+            password_request, list_id,
+            'Verifying the password', 'Wrong password')
+
     def _extract_videos(self, list_id, base_url):
         video_ids = []
         for pagenum in itertools.count(1):
+            page_url = self._page_url(base_url, pagenum)
             webpage = self._download_webpage(
-                self._page_url(base_url, pagenum), list_id,
+                page_url, list_id,
                 'Downloading page %s' % pagenum)
+
+            if pagenum == 1:
+                webpage = self._login_list_password(page_url, list_id, webpage)
+
             video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break
@@ -464,14 +499,24 @@ class VimeoAlbumIE(VimeoChannelIE):
             'title': 'Staff Favorites: November 2013',
         },
         'playlist_mincount': 13,
+    }, {
+        'note': 'Password-protected album',
+        'url': 'https://vimeo.com/album/3253534',
+        'info_dict': {
+            'title': 'test',
+            'id': '3253534',
+        },
+        'playlist_count': 1,
+        'params': {
+            'videopassword': 'youtube-dl',
+        }
     }]
 
     def _page_url(self, base_url, pagenum):
         return '%s/page:%d/' % (base_url, pagenum)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        album_id = mobj.group('id')
+        album_id = self._match_id(url)
         return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id)
 
 

From c0e1a415fdcd8d36d98a9fb6ff5bf9146004ddd3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:28:48 +0100
Subject: [PATCH 846/974] [firstpost] Modernize

---
 youtube_dl/extractor/firstpost.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/firstpost.py b/youtube_dl/extractor/firstpost.py
index 0993af1c9..298227d57 100644
--- a/youtube_dl/extractor/firstpost.py
+++ b/youtube_dl/extractor/firstpost.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -20,11 +18,10 @@ class FirstpostIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         page = self._download_webpage(url, video_id)
-        title = self._html_search_meta('twitter:title', page, 'title')
+
+        title = self._html_search_meta('twitter:title', page, 'title', fatal=True)
         description = self._html_search_meta('twitter:description', page, 'title')
 
         data = self._download_xml(
@@ -42,6 +39,7 @@ class FirstpostIE(InfoExtractor):
                 'height': int(details.find('./height').text.strip()),
             } for details in item.findall('./source/file_details') if details.find('./file').text
         ]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 6a78740211b37cab5d20e707db9f73dca6c60049 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:28:59 +0100
Subject: [PATCH 847/974] [test/test_youtube_signature] Use fake YDL

---
 test/test_youtube_signature.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 13d228cd8..09696e19a 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -8,11 +8,11 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 import io
 import re
 import string
 
+from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.compat import compat_str, compat_urlretrieve
 
@@ -88,7 +88,8 @@ def make_tfunc(url, stype, sig_input, expected_sig):
         if not os.path.exists(fn):
             compat_urlretrieve(url, fn)
 
-        ie = YoutubeIE()
+        ydl = FakeYDL()
+        ie = YoutubeIE(ydl)
         if stype == 'js':
             with io.open(fn, encoding='utf-8') as testf:
                 jscode = testf.read()

From 8f84f5718319293485eef7174702340f109c7227 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:42:41 +0100
Subject: [PATCH 848/974] [ccc] Add new extractor (Fixes #4890)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ccc.py      | 99 ++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 youtube_dl/extractor/ccc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb1e7f325..da3b2b2d0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -54,6 +54,7 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
+from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
new file mode 100644
index 000000000..2a5d4be18
--- /dev/null
+++ b/youtube_dl/extractor/ccc.py
@@ -0,0 +1,99 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    qualities,
+    unified_strdate,
+)
+
+
+class CCCIE(InfoExtractor):
+    IE_NAME = 'media.ccc.de'
+    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/[^?#]+/[^?#/]*?_(?P<id>[0-9]{8,})._[^?#/]*\.html'
+
+    _TEST = {
+        'url': 'http://media.ccc.de/browse/congress/2013/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor.html#video',
+        'md5': '205a365d0d57c0b1e43a12c9ffe8f9be',
+        'info_dict': {
+            'id': '20131228183',
+            'ext': 'mp4',
+            'title': 'Introduction to Processor Design',
+            'description': 'md5:5ddbf8c734800267f2cee4eab187bc1b',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'view_count': int,
+            'upload_date': '20131229',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        if self._downloader.params.get('prefer_free_formats'):
+            preference = qualities(['mp3', 'opus', 'mp4-lq', 'webm-lq', 'h264-sd', 'mp4-sd', 'webm-sd', 'mp4', 'webm', 'mp4-hd', 'h264-hd', 'webm-hd'])
+        else:
+            preference = qualities(['opus', 'mp3', 'webm-lq', 'mp4-lq', 'webm-sd', 'h264-sd', 'mp4-sd', 'webm', 'mp4', 'webm-hd', 'mp4-hd', 'h264-hd'])
+
+        title = self._html_search_regex(
+            r'(?s)<h1>(.*?)</h1>', webpage, 'title')
+        description = self._html_search_regex(
+            r"(?s)<p class='description'>(.*?)</p>",
+            webpage, 'description', fatal=False)
+        upload_date = unified_strdate(self._html_search_regex(
+            r"(?s)<span class='[^']*fa-calendar-o'></span>(.*?)</li>",
+            webpage, 'upload date', fatal=False))
+        view_count = int_or_none(self._html_search_regex(
+            r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>",
+            webpage, 'view count', fatal=False))
+
+        matches = re.finditer(r'''(?xs)
+            <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
+            <a\s+href='(?P<http_url>[^']+)'>\s*
+            (?:
+                .*?
+                <a\s+href='(?P<torrent_url>[^']+\.torrent)'
+            )?''', webpage)
+        formats = []
+        for m in matches:
+            format = m.group('format')
+            format_id = self._search_regex(
+                r'.*/([a-z0-9_-]+)/[^/]*$',
+                m.group('http_url'), 'format id', default=None)
+            vcodec = 'h264' if 'h264' in format_id else (
+                'none' if format_id in ('mp3', 'opus') else None
+            )
+            formats.append({
+                'format_id': format_id,
+                'format': format,
+                'url': m.group('http_url'),
+                'vcodec': vcodec,
+                'preference': preference(format_id),
+            })
+
+            if m.group('torrent_url'):
+                formats.append({
+                    'format_id': 'torrent-%s' % (format if format_id is None else format_id),
+                    'format': '%s (torrent)' % format,
+                    'proto': 'torrent',
+                    'format_note': '(unsupported; will just download the .torrent file)',
+                    'vcodec': vcodec,
+                    'preference': -100 + preference(format_id),
+                    'url': m.group('torrent_url'),
+                })
+        self._sort_formats(formats)
+
+        thumbnail = self._html_search_regex(
+            r"<video.*?poster='([^']+)'", webpage, 'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'view_count': view_count,
+            'upload_date': upload_date,
+            'formats': formats,
+        }

From 04bbe41330a4c8619e6fa3228d47c4e7b0396899 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:42:47 +0100
Subject: [PATCH 849/974] release 2015.02.10.3

---
 README.md              | 1 +
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7aad3007f..731cea1e1 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      on Windows)
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them.
+    --no-color                       Do not emit color codes in output.
 
 ## Network Options:
     --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 93572f6a8..8bce8fede 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -225,6 +225,7 @@
  - **mailru**: Видео@Mail.Ru
  - **Malemotion**
  - **MDR**
+ - **media.ccc.de**
  - **metacafe**
  - **Metacritic**
  - **Mgoon**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9ab5b49b8..13c436219 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.2'
+__version__ = '2015.02.10.3'

From 442c37b7a9a912cf3b0eb972335b7d24407f4537 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 11:28:28 +0100
Subject: [PATCH 850/974] [YoutubeDL] Do not perform filter matching on partial
 results (Fixes #4921)

---
 youtube_dl/YoutubeDL.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e2ec55c90..13d18e25e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -560,7 +560,7 @@ class YoutubeDL(object):
             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
             return None
 
-    def _match_entry(self, info_dict):
+    def _match_entry(self, info_dict, incomplete):
         """ Returns None iff the file should be downloaded """
 
         video_title = info_dict.get('title', info_dict.get('id', 'video'))
@@ -593,11 +593,12 @@ class YoutubeDL(object):
         if self.in_download_archive(info_dict):
             return '%s has already been recorded in archive' % video_title
 
-        match_filter = self.params.get('match_filter')
-        if match_filter is not None:
-            ret = match_filter(info_dict)
-            if ret is not None:
-                return ret
+        if not incomplete:
+            match_filter = self.params.get('match_filter')
+            if match_filter is not None:
+                ret = match_filter(info_dict)
+                if ret is not None:
+                    return ret
 
         return None
 
@@ -792,7 +793,7 @@ class YoutubeDL(object):
                     'extractor_key': ie_result['extractor_key'],
                 }
 
-                reason = self._match_entry(entry)
+                reason = self._match_entry(entry, incomplete=True)
                 if reason is not None:
                     self.to_screen('[download] ' + reason)
                     continue
@@ -1166,7 +1167,7 @@ class YoutubeDL(object):
         if 'format' not in info_dict:
             info_dict['format'] = info_dict['ext']
 
-        reason = self._match_entry(info_dict)
+        reason = self._match_entry(info_dict, incomplete=False)
         if reason is not None:
             self.to_screen('[download] ' + reason)
             return

From 612ee37365cf0e0e5c6a59fcce2986fa0754605c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 11:28:34 +0100
Subject: [PATCH 851/974] release 2015.02.10.4

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 13c436219..895d2da0d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.3'
+__version__ = '2015.02.10.4'

From 596ac6e31f6baaadf8280e2c54ca90a7cf2fbc2a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 15:45:36 +0100
Subject: [PATCH 852/974] [escapist] Modernize

---
 youtube_dl/extractor/escapist.py | 51 ++++++++++++++++----------------
 1 file changed, 25 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index e240cb859..98b698277 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -1,18 +1,17 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
     ExtractorError,
+    js_to_json,
 )
 
 
 class EscapistIE(InfoExtractor):
-    _VALID_URL = r'^https?://?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<id>[0-9]+)-'
+    _VALID_URL = r'https?://?(www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
     _TEST = {
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
         'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
@@ -20,31 +19,30 @@ class EscapistIE(InfoExtractor):
             'id': '6618',
             'ext': 'mp4',
             'description': "Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.",
-            'uploader': 'the-escapist-presents',
+            'uploader_id': 'the-escapist-presents',
+            'uploader': 'The Escapist Presents',
             'title': "Breaking Down Baldur's Gate",
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        showName = mobj.group('showname')
-        video_id = mobj.group('id')
-
-        self.report_extraction(video_id)
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        videoDesc = self._html_search_regex(
-            r'<meta name="description" content="([^"]*)"',
-            webpage, 'description', fatal=False)
+        uploader_id = self._html_search_regex(
+            r"<h1 class='headline'><a href='/videos/view/(.*?)'",
+            webpage, 'uploader ID', fatal=False)
+        uploader = self._html_search_regex(
+            r"<h1 class='headline'>(.*?)</a>",
+            webpage, 'uploader', fatal=False)
+        description = self._html_search_meta('description', webpage)
 
-        playerUrl = self._og_search_video_url(webpage, name='player URL')
+        raw_title = self._html_search_meta('title', webpage, fatal=True)
+        title = raw_title.partition(' : ')[2]
 
-        title = self._html_search_regex(
-            r'<meta name="title" content="([^"]*)"',
-            webpage, 'title').split(' : ')[-1]
-
-        configUrl = self._search_regex('config=(.*)$', playerUrl, 'config URL')
-        configUrl = compat_urllib_parse.unquote(configUrl)
+        player_url = self._og_search_video_url(webpage, name='player URL')
+        config_url = compat_urllib_parse.unquote(self._search_regex(
+            r'config=(.*)$', player_url, 'config URL'))
 
         formats = []
 
@@ -53,7 +51,7 @@ class EscapistIE(InfoExtractor):
                 cfgurl, video_id,
                 'Downloading ' + name + ' configuration',
                 'Unable to download ' + name + ' configuration',
-                transform_source=lambda s: s.replace("'", '"'))
+                transform_source=js_to_json)
 
             playlist = config['playlist']
             formats.append({
@@ -62,9 +60,9 @@ class EscapistIE(InfoExtractor):
                 'quality': quality,
             })
 
-        _add_format('normal', configUrl, quality=0)
-        hq_url = (configUrl +
-                  ('&hq=1' if '?' in configUrl else configUrl + '?hq=1'))
+        _add_format('normal', config_url, quality=0)
+        hq_url = (config_url +
+                  ('&hq=1' if '?' in config_url else config_url + '?hq=1'))
         try:
             _add_format('hq', hq_url, quality=1)
         except ExtractorError:
@@ -75,9 +73,10 @@ class EscapistIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'uploader': showName,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'description': videoDesc,
-            'player_url': playerUrl,
+            'description': description,
+            'player_url': player_url,
         }

From 9650885be954b3cc626205aa67f24c31f21da7b6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 15:55:51 +0100
Subject: [PATCH 853/974] [escapist] Filter video differently (Fixes #4919)

---
 youtube_dl/extractor/common.py   | 2 +-
 youtube_dl/extractor/escapist.py | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 382e846c5..8d71494b4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -271,7 +271,7 @@ class InfoExtractor(object):
             raise
         except compat_http_client.IncompleteRead as e:
             raise ExtractorError('A network error has occured.', cause=e, expected=True)
-        except (KeyError,) as e:
+        except (KeyError, StopIteration) as e:
             raise ExtractorError('An extractor error has occured.', cause=e)
 
     def set_downloader(self, downloader):
diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 98b698277..4303feccd 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -54,8 +54,11 @@ class EscapistIE(InfoExtractor):
                 transform_source=js_to_json)
 
             playlist = config['playlist']
+            video_url = next(
+                p['url'] for p in playlist
+                if p.get('eventCategory') == 'Video')
             formats.append({
-                'url': playlist[1]['url'],
+                'url': video_url,
                 'format_id': name,
                 'quality': quality,
             })

From af0d11f2441728ce12ba8a424de29aa873b12d86 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 15:56:04 +0100
Subject: [PATCH 854/974] release 2015.02.10.5

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 895d2da0d..225b30f8c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.4'
+__version__ = '2015.02.10.5'

From 054fe3cc40cc0feaa69e21b31946c568a749eb76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 10 Feb 2015 21:35:34 +0600
Subject: [PATCH 855/974] [ntvru] Adapt to new direct delivery and modernize
 (Closes #4918)

---
 youtube_dl/extractor/ntvru.py | 97 +++++++++++++++--------------------
 1 file changed, 40 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 0ab8d5100..2cd924d05 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -3,7 +3,9 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    unescapeHTML
+    clean_html,
+    xpath_text,
+    int_or_none,
 )
 
 
@@ -14,73 +16,63 @@ class NTVRuIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.ntv.ru/novosti/863142/',
+            'md5': 'ba7ea172a91cb83eb734cad18c10e723',
             'info_dict': {
                 'id': '746000',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
                 'description': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 136,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/video/novosti/750370/',
+            'md5': 'adecff79691b4d71e25220a191477124',
             'info_dict': {
                 'id': '750370',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
                 'description': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 172,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
+            'md5': '82dbd49b38e3af1d00df16acbeab260c',
             'info_dict': {
                 'id': '747480',
-                'ext': 'flv',
-                'title': '«Сегодня». 21 марта 2014 года. 16:00 ',
-                'description': '«Сегодня». 21 марта 2014 года. 16:00 ',
+                'ext': 'mp4',
+                'title': '«Сегодня». 21 марта 2014 года. 16:00',
+                'description': '«Сегодня». 21 марта 2014 года. 16:00',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 1496,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/kino/Koma_film',
+            'md5': 'f825770930937aa7e5aca0dc0d29319a',
             'info_dict': {
-                'id': '758100',
-                'ext': 'flv',
+                'id': '1007609',
+                'ext': 'mp4',
                 'title': 'Остросюжетный фильм «Кома»',
                 'description': 'Остросюжетный фильм «Кома»',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 5592,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
+            'md5': '9320cd0e23f3ea59c330dc744e06ff3b',
             'info_dict': {
                 'id': '751482',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': '«Дело врачей»: «Деревце жизни»',
                 'description': '«Дело врачей»: «Деревце жизни»',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 2590,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
     ]
 
@@ -92,45 +84,36 @@ class NTVRuIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
 
-        video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, page, 'video id')
+        webpage = self._download_webpage(url, video_id)
 
-        player = self._download_xml('http://www.ntv.ru/vi%s/' % video_id, video_id, 'Downloading video XML')
-        title = unescapeHTML(player.find('./data/title').text)
-        description = unescapeHTML(player.find('./data/description').text)
+        video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, webpage, 'video id')
+
+        player = self._download_xml(
+            'http://www.ntv.ru/vi%s/' % video_id,
+            video_id, 'Downloading video XML')
+        title = clean_html(xpath_text(player, './data/title', 'title', fatal=True))
+        description = clean_html(xpath_text(player, './data/description', 'description'))
 
         video = player.find('./data/video')
-        video_id = video.find('./id').text
-        thumbnail = video.find('./splash').text
-        duration = int(video.find('./totaltime').text)
-        view_count = int(video.find('./views').text)
-        puid22 = video.find('./puid22').text
+        video_id = xpath_text(video, './id', 'video id')
+        thumbnail = xpath_text(video, './splash', 'thumbnail')
+        duration = int_or_none(xpath_text(video, './totaltime', 'duration'))
+        view_count = int_or_none(xpath_text(video, './views', 'view count'))
 
-        apps = {
-            '4': 'video1',
-            '7': 'video2',
-        }
-
-        app = apps.get(puid22, apps['4'])
+        token = self._download_webpage(
+            'http://stat.ntv.ru/services/access/token',
+            video_id, 'Downloading access token')
 
         formats = []
         for format_id in ['', 'hi', 'webm']:
-            file = video.find('./%sfile' % format_id)
-            if file is None:
+            file_ = video.find('./%sfile' % format_id)
+            if file_ is None:
                 continue
             size = video.find('./%ssize' % format_id)
             formats.append({
-                'url': 'rtmp://media.ntv.ru/%s' % app,
-                'app': app,
-                'play_path': file.text,
-                'rtmp_conn': 'B:1',
-                'player_url': 'http://www.ntv.ru/swf/vps1.swf?update=20131128',
-                'page_url': 'http://www.ntv.ru',
-                'flash_version': 'LNX 11,2,202,341',
-                'rtmp_live': True,
-                'ext': 'flv',
-                'filesize': int(size.text),
+                'url': 'http://media2.ntv.ru/vod/%s&tok=%s' % (file_.text, token),
+                'filesize': int_or_none(size.text if size is not None else None),
             })
         self._sort_formats(formats)
 

From 8708d764252a83e650f36fe47f89ace4f2a91a5b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Feb 2015 16:39:15 +0800
Subject: [PATCH 856/974] [camdemy] Add new extractor

Single file download done, while folder extaction in plan
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/camdemy.py  | 78 ++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/camdemy.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index da3b2b2d0..496e4d0b7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -49,6 +49,7 @@ from .brightcove import BrightcoveIE
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
+from .camdemy import CamdemyIE
 from .canal13cl import Canal13clIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
new file mode 100644
index 000000000..54959f2cf
--- /dev/null
+++ b/youtube_dl/extractor/camdemy.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import parse_iso8601
+
+
+class CamdemyIE(InfoExtractor):
+    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+).*'
+    _TESTS = [{
+        # single file
+        'url': 'http://www.camdemy.com/media/5181/',
+        'md5': '5a5562b6a98b37873119102e052e311b',
+        'info_dict': {
+            'id': '5181',
+            'ext': 'mp4',
+            'title': 'Ch1-1 Introduction, Signals (02-23-2012)',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': '',
+            'creator': 'ss11spring',
+            'upload_date': '20130114',
+            'timestamp': 1358154556,
+        }
+    }, {
+        # With non-empty description
+        'url': 'http://www.camdemy.com/media/13885',
+        'md5': '4576a3bb2581f86c61044822adbd1249',
+        'info_dict': {
+            'id': '13885',
+            'ext': 'mp4',
+            'title': 'EverCam + Camdemy QuickStart',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:050b62f71ed62928f8a35f1a41e186c9',
+            'creator': 'evercam',
+            'upload_date': '20140620',
+            'timestamp': 1403271569,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        page = self._download_webpage(url, video_id)
+
+        oembed_obj = self._download_json(
+            'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
+
+        thumb_url = oembed_obj['thumbnail_url']
+        video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
+        fileListXML = self._download_xml(
+            compat_urlparse.urljoin(video_folder, 'fileList.xml'),
+            video_id, 'Filelist XML')
+        fileName = fileListXML.find('./video/item/fileName').text
+
+        creation_time = self._html_search_regex(
+            r"<div class='title'>Posted :</div>.*<div class='value'>([0-9:\- ]+)<",
+            page, 'creation time', flags=re.MULTILINE | re.DOTALL) + '+08:00'
+        creation_timestamp = parse_iso8601(creation_time, delimiter=' ')
+
+        view_count_str = self._html_search_regex(
+            r"<div class='title'>Views :</div>.*<div class='value'>([0-9,]+)<",
+            page, 'view count', flags=re.MULTILINE | re.DOTALL)
+        views = int(view_count_str.replace(',', ''))
+
+        return {
+            'id': video_id,
+            'url': compat_urlparse.urljoin(video_folder, fileName),
+            'title': oembed_obj['title'],
+            'thumbnail': thumb_url,
+            'description': self._html_search_meta('description', page),
+            'creator': oembed_obj['author_name'],
+            'duration': oembed_obj['duration'],
+            'timestamp': creation_timestamp,
+            'view_count': views,
+        }

From 3811c567e734f38f830e8c2e0a45afa7c48cbf12 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 11 Feb 2015 15:47:19 +0200
Subject: [PATCH 857/974] [teamcoco] Fix video id extraction

---
 youtube_dl/extractor/teamcoco.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index e85d452a3..a73da1c9c 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -30,6 +30,11 @@ class TeamcocoIE(InfoExtractor):
             }
         }
     ]
+    _VIDEO_ID_REGEXES = (
+        r'"eVar42"\s*:\s*(\d+)',
+        r'Ginger\.TeamCoco\.openInApp\("video",\s*"([^"]+)"',
+        r'"id_not"\s*:\s*(\d+)'
+    )
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -40,8 +45,7 @@ class TeamcocoIE(InfoExtractor):
         video_id = mobj.group("video_id")
         if not video_id:
             video_id = self._html_search_regex(
-                r'<div\s+class="player".*?data-id="(\d+?)"',
-                webpage, 'video id')
+                self._VIDEO_ID_REGEXES, webpage, 'video id')
 
         data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
         data = self._download_xml(

From ea5db8469ee3e5f1bb4ba3deb88f28b4d9838e09 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 11 Feb 2015 16:21:52 +0200
Subject: [PATCH 858/974] [canalplus] Add support for itele.fr URLs (Closes
 #4931)

---
 youtube_dl/extractor/canalplus.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 11d18d74a..1b14471e5 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -15,12 +15,13 @@ from ..utils import (
 
 class CanalplusIE(InfoExtractor):
     IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
-    _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
+    _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv|itele\.fr)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
     _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s'
     _SITE_ID_MAP = {
         'canalplus.fr': 'cplus',
         'piwiplus.fr': 'teletoon',
         'd8.tv': 'd8',
+        'itele.fr': 'itele',
     }
 
     _TESTS = [{
@@ -53,6 +54,16 @@ class CanalplusIE(InfoExtractor):
             'upload_date': '20131108',
         },
         'skip': 'videos get deleted after a while',
+    }, {
+        'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
+        'md5': '65aa83ad62fe107ce29e564bb8712580',
+        'info_dict': {
+            'id': '1213714',
+            'ext': 'flv',
+            'title': 'Aubervilliers : un lycée en colère - Le 11/02/2015 à 06h45',
+            'description': 'md5:8216206ec53426ea6321321f3b3c16db',
+            'upload_date': '20150211',
+        },
     }]
 
     def _real_extract(self, url):

From c56d7d899d5587cd73cb017b776ceb302494b6fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Feb 2015 22:10:33 +0600
Subject: [PATCH 859/974] [dctptv] Skip rtmp download

---
 youtube_dl/extractor/dctp.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 6ed354379..aa2c09eb6 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -14,6 +14,10 @@ class DctpTvIE(InfoExtractor):
             'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
             'ext': 'flv',
             'title': 'Videoinstallation für eine Kaufhausfassade'
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
         }
     }
 

From 8367d3f3cbad7ddae5116c3cd5d39ebfed9711c5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Feb 2015 00:11:33 +0800
Subject: [PATCH 860/974] [camdemy] Detection of external sources

---
 youtube_dl/extractor/camdemy.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 54959f2cf..9a6341f0f 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -38,6 +38,19 @@ class CamdemyIE(InfoExtractor):
             'upload_date': '20140620',
             'timestamp': 1403271569,
         }
+    }, {
+        # External source
+        'url': 'http://www.camdemy.com/media/14842',
+        'md5': '50e1c3c3aa233d3d7b7daa2fa10b1cf7',
+        'info_dict': {
+            'id': '2vsYQzNIsJo',
+            'ext': 'mp4',
+            'upload_date': '20130211',
+            'uploader': 'Hun Kim',
+            'description': 'Excel 2013 Tutorial for Beginners - How to add Password Protection',
+            'uploader_id': 'hunkimtutorials',
+            'title': 'Excel 2013 Tutorial - How to add Password Protection',
+        }
     }]
 
     def _real_extract(self, url):
@@ -45,6 +58,13 @@ class CamdemyIE(InfoExtractor):
 
         page = self._download_webpage(url, video_id)
 
+        srcFrom = self._html_search_regex(
+            r"<div class='srcFrom'>Source: <a title='([^']+)'", page,
+            'external source', default=None)
+
+        if srcFrom:
+            return self.url_result(srcFrom)
+
         oembed_obj = self._download_json(
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 

From d304209a850793cd121820307be9bee7e9b9ae1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 17:25:04 +0100
Subject: [PATCH 861/974] [test/parameters.json] Set 'fixup' to 'never'

The fixed audio files for Youtube have a size lower that the minimum required.
---
 test/parameters.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/parameters.json b/test/parameters.json
index 098cd0cd0..af77b89b4 100644
--- a/test/parameters.json
+++ b/test/parameters.json
@@ -39,5 +39,6 @@
     "writesubtitles": false,
     "allsubtitles": false,
     "listssubtitles": false,
-    "socket_timeout": 20
+    "socket_timeout": 20,
+    "fixup": "never"
 }

From 124f3bc67d61adeb4dc257eb8a678df821c27314 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Feb 2015 22:33:03 +0600
Subject: [PATCH 862/974] [dotsub] Fix extraction and modernize

---
 youtube_dl/extractor/dotsub.py | 44 +++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py
index 638bb33cd..f51d88a98 100644
--- a/youtube_dl/extractor/dotsub.py
+++ b/youtube_dl/extractor/dotsub.py
@@ -1,13 +1,14 @@
 from __future__ import unicode_literals
 
-import re
-import time
-
 from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
 
 
 class DotsubIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?dotsub\.com/view/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dotsub\.com/view/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://dotsub.com/view/aed3b8b2-1889-4df5-ae63-ad85f5572f27',
         'md5': '0914d4d69605090f623b7ac329fea66e',
@@ -15,28 +16,37 @@ class DotsubIE(InfoExtractor):
             'id': 'aed3b8b2-1889-4df5-ae63-ad85f5572f27',
             'ext': 'flv',
             'title': 'Pyramids of Waste (2010), AKA The Lightbulb Conspiracy - Planned obsolescence documentary',
+            'description': 'md5:699a0f7f50aeec6042cb3b1db2d0d074',
+            'thumbnail': 're:^https?://dotsub.com/media/aed3b8b2-1889-4df5-ae63-ad85f5572f27/p',
+            'duration': 3169,
             'uploader': '4v4l0n42',
-            'description': 'Pyramids of Waste (2010) also known as "The lightbulb conspiracy" is a documentary about how our economic system based on consumerism  and planned obsolescence is breaking our planet down.\r\n\r\nSolutions to this can be found at:\r\nhttp://robotswillstealyourjob.com\r\nhttp://www.federicopistono.org\r\n\r\nhttp://opensourceecology.org\r\nhttp://thezeitgeistmovement.com',
-            'thumbnail': 'http://dotsub.com/media/aed3b8b2-1889-4df5-ae63-ad85f5572f27/p',
+            'timestamp': 1292248482.625,
             'upload_date': '20101213',
+            'view_count': int,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        info_url = "https://dotsub.com/api/media/%s/metadata" % video_id
-        info = self._download_json(info_url, video_id)
-        date = time.gmtime(info['dateCreated'] / 1000)  # The timestamp is in miliseconds
+        video_id = self._match_id(url)
+
+        info = self._download_json(
+            'https://dotsub.com/api/media/%s/metadata' % video_id, video_id)
+        video_url = info.get('mediaURI')
+
+        if not video_url:
+            webpage = self._download_webpage(url, video_id)
+            video_url = self._search_regex(
+                r'"file"\s*:\s*\'([^\']+)', webpage, 'video url')
 
         return {
             'id': video_id,
-            'url': info['mediaURI'],
+            'url': video_url,
             'ext': 'flv',
             'title': info['title'],
-            'thumbnail': info['screenshotURI'],
-            'description': info['description'],
-            'uploader': info['user'],
-            'view_count': info['numberOfViews'],
-            'upload_date': '%04i%02i%02i' % (date.tm_year, date.tm_mon, date.tm_mday),
+            'description': info.get('description'),
+            'thumbnail': info.get('screenshotURI'),
+            'duration': int_or_none(info.get('duration'), 1000),
+            'uploader': info.get('user'),
+            'timestamp': float_or_none(info.get('dateCreated'), 1000),
+            'view_count': int_or_none(info.get('numberOfViews')),
         }

From f36f92f4da3664f7286b8b4327ac5791fb98e5b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 18:07:12 +0100
Subject: [PATCH 863/974] [aes] style: Put __all__ variable at the end of the
 file

---
 youtube_dl/aes.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 5efd0f836..07224d508 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text']
-
 import base64
 from math import ceil
 
@@ -329,3 +327,5 @@ def inc(data):
             data[i] = data[i] + 1
             break
     return data
+
+__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text']

From 050fa4356122926d673635f7ba343293126e6d5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 18:08:42 +0100
Subject: [PATCH 864/974] flake8: Ignore some error added in pep8 1.6

* E402: we exectute code between imports, like modifying 'sys.path' in the tests
* E731: we assign to lambdas in a lot of places, we may want to consider defining functions in a single line instead (what pep8 recommends)
---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 02aa56257..26857750c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -3,4 +3,4 @@ universal = True
 
 [flake8]
 exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build,.git
-ignore = E501
+ignore = E402,E501,E731

From 2d30521ab9f535559dd208293982138516b88e8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 18:39:31 +0100
Subject: [PATCH 865/974] [youtube] Extract average rating (closes #2362)

---
 youtube_dl/extractor/common.py  | 1 +
 youtube_dl/extractor/youtube.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8d71494b4..48742189a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -157,6 +157,7 @@ class InfoExtractor(object):
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video
+    average_rating: Average rating give by users, the scale used depends on the webpage
     comment_count:  Number of comments on the video
     comments:       A list of comments, each with one or more of the following
                     properties (all but one of text or html optional):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8b91fff18..35ef4c303 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -25,6 +25,7 @@ from ..compat import (
 from ..utils import (
     clean_html,
     ExtractorError,
+    float_or_none,
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
@@ -1124,6 +1125,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
             'dislike_count': dislike_count,
+            'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
             'formats': formats,
         }
 

From 5277f09dfce6ba1a6f191871df1515ddd68ddd42 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 11 Feb 2015 19:02:39 +0100
Subject: [PATCH 866/974] release 2015.02.11

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 225b30f8c..492ddf2ea 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.5'
+__version__ = '2015.02.11'

From c40feaba77d8717243467a65fcce4dcab6dfe7bf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Feb 2015 14:13:19 +0800
Subject: [PATCH 867/974] [camdemy] Add support for folders

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/camdemy.py  | 68 +++++++++++++++++++++++++++-----
 2 files changed, 63 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 496e4d0b7..095feabf9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -49,7 +49,10 @@ from .brightcove import BrightcoveIE
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
-from .camdemy import CamdemyIE
+from .camdemy import (
+    CamdemyIE,
+    CamdemyFolderIE
+)
 from .canal13cl import Canal13clIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 9a6341f0f..0d64b5a16 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -4,12 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import compat_urllib_parse
 from ..utils import parse_iso8601
 
 
 class CamdemyIE(InfoExtractor):
-    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+).*'
+    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+)'
     _TESTS = [{
         # single file
         'url': 'http://www.camdemy.com/media/5181/',
@@ -69,25 +69,25 @@ class CamdemyIE(InfoExtractor):
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 
         thumb_url = oembed_obj['thumbnail_url']
-        video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
+        video_folder = compat_urllib_parse.urljoin(thumb_url, 'video/')
         fileListXML = self._download_xml(
-            compat_urlparse.urljoin(video_folder, 'fileList.xml'),
+            compat_urllib_parse.urljoin(video_folder, 'fileList.xml'),
             video_id, 'Filelist XML')
         fileName = fileListXML.find('./video/item/fileName').text
 
         creation_time = self._html_search_regex(
-            r"<div class='title'>Posted :</div>.*<div class='value'>([0-9:\- ]+)<",
-            page, 'creation time', flags=re.MULTILINE | re.DOTALL) + '+08:00'
+            r"<div class='title'>Posted :</div>[\r\n ]*<div class='value'>([^<>]+)<",
+            page, 'creation time', flags=re.MULTILINE) + '+08:00'
         creation_timestamp = parse_iso8601(creation_time, delimiter=' ')
 
         view_count_str = self._html_search_regex(
-            r"<div class='title'>Views :</div>.*<div class='value'>([0-9,]+)<",
-            page, 'view count', flags=re.MULTILINE | re.DOTALL)
+            r"<div class='title'>Views :</div>[\r\n ]*<div class='value'>([^<>]+)<",
+            page, 'view count', flags=re.MULTILINE)
         views = int(view_count_str.replace(',', ''))
 
         return {
             'id': video_id,
-            'url': compat_urlparse.urljoin(video_folder, fileName),
+            'url': compat_urllib_parse.urljoin(video_folder, fileName),
             'title': oembed_obj['title'],
             'thumbnail': thumb_url,
             'description': self._html_search_meta('description', page),
@@ -96,3 +96,53 @@ class CamdemyIE(InfoExtractor):
             'timestamp': creation_timestamp,
             'view_count': views,
         }
+
+
+class CamdemyFolderIE(InfoExtractor):
+    _VALID_URL = r'http://www.camdemy.com/folder/(?P<id>\d+)'
+    _TESTS = [{
+        # links with trailing slash
+        'url': 'http://www.camdemy.com/folder/450',
+        'info_dict': {
+            'id': '450',
+            'title': '信號與系統 2012 & 2011 (Signals and Systems)',
+        },
+        'playlist_mincount': 145
+    }, {
+        # links without trailing slash
+        # and multi-page
+        'url': 'http://www.camdemy.com/folder/853',
+        'info_dict': {
+            'id': '853',
+            'title': '科學計算 - 使用 Matlab'
+        },
+        'playlist_mincount': 20
+    }, {
+        # with displayMode parameter. For testing the codes to add parameters
+        'url': 'http://www.camdemy.com/folder/853/?displayMode=defaultOrderByOrg',
+        'info_dict': {
+            'id': '853',
+            'title': '科學計算 - 使用 Matlab'
+        },
+        'playlist_mincount': 20
+    }]
+
+    def _real_extract(self, url):
+        folder_id = self._match_id(url)
+
+        # Add displayMode=list so that all links are displayed in a single page
+        parsed_url = list(compat_urllib_parse.urlparse(url))
+        query = dict(compat_urllib_parse.parse_qsl(parsed_url[4]))
+        query.update({'displayMode': 'list'})
+        parsed_url[4] = compat_urllib_parse.urlencode(query)
+        final_url = compat_urllib_parse.urlunparse(parsed_url)
+
+        page = self._download_webpage(final_url, folder_id)
+        matches = re.findall(r"href='(/media/\d+/?)'", page)
+
+        entries = [self.url_result('http://www.camdemy.com' + media_path)
+                   for media_path in matches]
+
+        folder_title = self._html_search_meta('keywords', page)
+
+        return self.playlist_result(entries, folder_id, folder_title)

From 7e6011101f00e4f7216467e80091cc89c2846f01 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Feb 2015 14:23:25 +0800
Subject: [PATCH 868/974] [camdemy] Python2 compatibility

---
 youtube_dl/extractor/camdemy.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 0d64b5a16..1bc602c31 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (compat_urllib_parse, compat_urlparse)
 from ..utils import parse_iso8601
 
 
@@ -69,9 +69,9 @@ class CamdemyIE(InfoExtractor):
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 
         thumb_url = oembed_obj['thumbnail_url']
-        video_folder = compat_urllib_parse.urljoin(thumb_url, 'video/')
+        video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
         fileListXML = self._download_xml(
-            compat_urllib_parse.urljoin(video_folder, 'fileList.xml'),
+            compat_urlparse.urljoin(video_folder, 'fileList.xml'),
             video_id, 'Filelist XML')
         fileName = fileListXML.find('./video/item/fileName').text
 
@@ -87,7 +87,7 @@ class CamdemyIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': compat_urllib_parse.urljoin(video_folder, fileName),
+            'url': compat_urlparse.urljoin(video_folder, fileName),
             'title': oembed_obj['title'],
             'thumbnail': thumb_url,
             'description': self._html_search_meta('description', page),
@@ -131,11 +131,11 @@ class CamdemyFolderIE(InfoExtractor):
         folder_id = self._match_id(url)
 
         # Add displayMode=list so that all links are displayed in a single page
-        parsed_url = list(compat_urllib_parse.urlparse(url))
-        query = dict(compat_urllib_parse.parse_qsl(parsed_url[4]))
+        parsed_url = list(compat_urlparse.urlparse(url))
+        query = dict(compat_urlparse.parse_qsl(parsed_url[4]))
         query.update({'displayMode': 'list'})
         parsed_url[4] = compat_urllib_parse.urlencode(query)
-        final_url = compat_urllib_parse.urlunparse(parsed_url)
+        final_url = compat_urlparse.urlunparse(parsed_url)
 
         page = self._download_webpage(final_url, folder_id)
         matches = re.findall(r"href='(/media/\d+/?)'", page)

From 08b38d54015706d90d05371455ba86b7887cabcc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 12 Feb 2015 08:55:06 +0100
Subject: [PATCH 869/974] [camdemy] Simplify and make more robust (#4938)

Do not throw errors if view count or upload date extraction fails.
Dispose of re.MULTILINE, which had absolutely no effect without any ^ or $ in sight.
Follow PEP8 naming conventions.
---
 youtube_dl/extractor/camdemy.py | 47 ++++++++++++++++++---------------
 youtube_dl/utils.py             | 27 ++++++++++---------
 2 files changed, 40 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 1bc602c31..5de5879b4 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -1,11 +1,18 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import datetime
 import re
 
 from .common import InfoExtractor
-from ..compat import (compat_urllib_parse, compat_urlparse)
-from ..utils import parse_iso8601
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
+from ..utils import (
+    parse_iso8601,
+    str_to_int,
+)
 
 
 class CamdemyIE(InfoExtractor):
@@ -23,6 +30,7 @@ class CamdemyIE(InfoExtractor):
             'creator': 'ss11spring',
             'upload_date': '20130114',
             'timestamp': 1358154556,
+            'view_count': int,
         }
     }, {
         # With non-empty description
@@ -55,46 +63,43 @@ class CamdemyIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         page = self._download_webpage(url, video_id)
 
-        srcFrom = self._html_search_regex(
+        src_from = self._html_search_regex(
             r"<div class='srcFrom'>Source: <a title='([^']+)'", page,
             'external source', default=None)
-
-        if srcFrom:
-            return self.url_result(srcFrom)
+        if src_from:
+            return self.url_result(src_from)
 
         oembed_obj = self._download_json(
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 
         thumb_url = oembed_obj['thumbnail_url']
         video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
-        fileListXML = self._download_xml(
+        file_list_doc = self._download_xml(
             compat_urlparse.urljoin(video_folder, 'fileList.xml'),
             video_id, 'Filelist XML')
-        fileName = fileListXML.find('./video/item/fileName').text
+        file_name = file_list_doc.find('./video/item/fileName').text
+        video_url = compat_urlparse.urljoin(video_folder, file_name)
 
-        creation_time = self._html_search_regex(
-            r"<div class='title'>Posted :</div>[\r\n ]*<div class='value'>([^<>]+)<",
-            page, 'creation time', flags=re.MULTILINE) + '+08:00'
-        creation_timestamp = parse_iso8601(creation_time, delimiter=' ')
-
-        view_count_str = self._html_search_regex(
-            r"<div class='title'>Views :</div>[\r\n ]*<div class='value'>([^<>]+)<",
-            page, 'view count', flags=re.MULTILINE)
-        views = int(view_count_str.replace(',', ''))
+        timestamp = parse_iso8601(self._html_search_regex(
+            r"<div class='title'>Posted\s*:</div>\s*<div class='value'>([^<>]+)<",
+            page, 'creation time', fatal=False),
+            delimiter=' ', timezone=datetime.timedelta(hours=8))
+        view_count = str_to_int(self._html_search_regex(
+            r"<div class='title'>Views\s*:</div>\s*<div class='value'>([^<>]+)<",
+            page, 'view count', fatal=False))
 
         return {
             'id': video_id,
-            'url': compat_urlparse.urljoin(video_folder, fileName),
+            'url': video_url,
             'title': oembed_obj['title'],
             'thumbnail': thumb_url,
             'description': self._html_search_meta('description', page),
             'creator': oembed_obj['author_name'],
             'duration': oembed_obj['duration'],
-            'timestamp': creation_timestamp,
-            'view_count': views,
+            'timestamp': timestamp,
+            'view_count': view_count,
         }
 
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 03566d223..54fa17c38 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -666,26 +666,27 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
             req, **kwargs)
 
 
-def parse_iso8601(date_str, delimiter='T'):
+def parse_iso8601(date_str, delimiter='T', timezone=None):
     """ Return a UNIX timestamp from the given date """
 
     if date_str is None:
         return None
 
-    m = re.search(
-        r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
-        date_str)
-    if not m:
-        timezone = datetime.timedelta()
-    else:
-        date_str = date_str[:-len(m.group(0))]
-        if not m.group('sign'):
+    if timezone is None:
+        m = re.search(
+            r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
+            date_str)
+        if not m:
             timezone = datetime.timedelta()
         else:
-            sign = 1 if m.group('sign') == '+' else -1
-            timezone = datetime.timedelta(
-                hours=sign * int(m.group('hours')),
-                minutes=sign * int(m.group('minutes')))
+            date_str = date_str[:-len(m.group(0))]
+            if not m.group('sign'):
+                timezone = datetime.timedelta()
+            else:
+                sign = 1 if m.group('sign') == '+' else -1
+                timezone = datetime.timedelta(
+                    hours=sign * int(m.group('hours')),
+                    minutes=sign * int(m.group('minutes')))
     date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
     dt = datetime.datetime.strptime(date_str, date_format) - timezone
     return calendar.timegm(dt.timetuple())

From 7c24ce225d1ddb350f91b0fd78fefa40f6aa4481 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 12 Feb 2015 19:19:55 +0100
Subject: [PATCH 870/974] [NPORadio] Added extractor for live radio

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nporadio.py | 40 ++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 youtube_dl/extractor/nporadio.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 095feabf9..b118c3d1d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -320,6 +320,7 @@ from .npo import (
     NPOLiveIE,
     TegenlichtVproIE,
 )
+from .nporadio import NPORadioIE
 from .nrk import (
     NRKIE,
     NRKTVIE,
diff --git a/youtube_dl/extractor/nporadio.py b/youtube_dl/extractor/nporadio.py
new file mode 100644
index 000000000..dd8d97cb7
--- /dev/null
+++ b/youtube_dl/extractor/nporadio.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+
+
+class NPORadioIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/(?P<id>.*)'
+    _TEST = {
+        'url': 'http://www.npo.nl/radio/radio-1',
+        'info_dict': {
+            'id': 'radio-1',
+            'ext': 'mp3',
+            'title': 'NPO Radio 1',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+                self._html_get_attribute_regex('data-channel'), webpage, 'title')
+       
+        json_data = json.loads(
+                     self._html_search_regex(
+                     self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
+        
+        return {
+            'id': video_id,
+            'title': title,
+            'ext': json_data['codec'],
+            'url': json_data['url']
+        }
+
+    def _html_get_attribute_regex(self, attribute):
+        return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)
+

From c3d64fc1b30b46cc4f58aaa55c02918248bb1bee Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 12 Feb 2015 19:27:59 +0100
Subject: [PATCH 871/974] [nporadio] Edit to confirm to flake8 standards

---
 youtube_dl/extractor/nporadio.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nporadio.py b/youtube_dl/extractor/nporadio.py
index dd8d97cb7..d1c4ec167 100644
--- a/youtube_dl/extractor/nporadio.py
+++ b/youtube_dl/extractor/nporadio.py
@@ -22,12 +22,12 @@ class NPORadioIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
-                self._html_get_attribute_regex('data-channel'), webpage, 'title')
-       
+            self._html_get_attribute_regex('data-channel'), webpage, 'title')
+
         json_data = json.loads(
-                     self._html_search_regex(
-                     self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
-        
+            self._html_search_regex(
+                self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
+
         return {
             'id': video_id,
             'title': title,
@@ -37,4 +37,3 @@ class NPORadioIE(InfoExtractor):
 
     def _html_get_attribute_regex(self, attribute):
         return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)
-

From 171ca612af948b2966e56054ce3a25a9f5ab2a76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Feb 2015 01:36:54 +0600
Subject: [PATCH 872/974] [npo:radio] Move to extractor to common npo place and
 add extractor for fragments

---
 youtube_dl/extractor/__init__.py |  3 +-
 youtube_dl/extractor/npo.py      | 83 +++++++++++++++++++++++++++++++-
 youtube_dl/extractor/nporadio.py | 39 ---------------
 3 files changed, 83 insertions(+), 42 deletions(-)
 delete mode 100644 youtube_dl/extractor/nporadio.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b118c3d1d..a4fab540b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -318,9 +318,10 @@ from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
     NPOLiveIE,
+    NPORadioIE,
+    NPORadioFragmentIE,
     TegenlichtVproIE,
 )
-from .nporadio import NPORadioIE
 from .nrk import (
     NRKIE,
     NRKTVIE,
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 54be06a4e..c075618e8 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
@@ -22,7 +23,7 @@ class NPOBaseIE(SubtitlesInfoExtractor):
 
 class NPOIE(NPOBaseIE):
     IE_NAME = 'npo.nl'
-    _VALID_URL = r'https?://www\.npo\.nl/[^/]+/[^/]+/(?P<id>[^/?]+)'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/(?!live|radio)[^/]+/[^/]+/(?P<id>[^/?]+)'
 
     _TESTS = [
         {
@@ -185,7 +186,7 @@ class NPOIE(NPOBaseIE):
 
 class NPOLiveIE(NPOBaseIE):
     IE_NAME = 'npo.nl:live'
-    _VALID_URL = r'https?://www\.npo\.nl/live/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live/(?P<id>.+)'
 
     _TEST = {
         'url': 'http://www.npo.nl/live/npo-1',
@@ -260,6 +261,84 @@ class NPOLiveIE(NPOBaseIE):
         }
 
 
+class NPORadioIE(InfoExtractor):
+    IE_NAME = 'npo.nl:radio'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/(?P<id>[^/]+)/?$'
+
+    _TEST = {
+        'url': 'http://www.npo.nl/radio/radio-1',
+        'info_dict': {
+            'id': 'radio-1',
+            'ext': 'mp3',
+            'title': 're:^NPO Radio 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    @staticmethod
+    def _html_get_attribute_regex(attribute):
+        return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            self._html_get_attribute_regex('data-channel'), webpage, 'title')
+
+        stream = self._parse_json(
+            self._html_search_regex(self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'),
+            video_id)
+
+        codec = stream.get('codec')
+
+        return {
+            'id': video_id,
+            'url': stream['url'],
+            'title': self._live_title(title),
+            'acodec': codec,
+            'ext': codec,
+            'is_live': True,
+        }
+
+
+class NPORadioFragmentIE(InfoExtractor):
+    IE_NAME = 'npo.nl:radio:fragment'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/[^/]+/fragment/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://www.npo.nl/radio/radio-5/fragment/174356',
+        'md5': 'dd8cc470dad764d0fdc70a9a1e2d18c2',
+        'info_dict': {
+            'id': '174356',
+            'ext': 'mp3',
+            'title': 'Jubileumconcert Willeke Alberti',
+        },
+    }
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, audio_id)
+
+        title = self._html_search_regex(
+            r'href="/radio/[^/]+/fragment/%s" title="([^"]+)"' % audio_id,
+            webpage, 'title')
+
+        audio_url = self._search_regex(
+            r"data-streams='([^']+)'", webpage, 'audio url')
+
+        return {
+            'id': audio_id,
+            'url': audio_url,
+            'title': title,
+        }
+
+
 class TegenlichtVproIE(NPOIE):
     IE_NAME = 'tegenlicht.vpro.nl'
     _VALID_URL = r'https?://tegenlicht\.vpro\.nl/afleveringen/.*?'
diff --git a/youtube_dl/extractor/nporadio.py b/youtube_dl/extractor/nporadio.py
deleted file mode 100644
index d1c4ec167..000000000
--- a/youtube_dl/extractor/nporadio.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-
-
-class NPORadioIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/(?P<id>.*)'
-    _TEST = {
-        'url': 'http://www.npo.nl/radio/radio-1',
-        'info_dict': {
-            'id': 'radio-1',
-            'ext': 'mp3',
-            'title': 'NPO Radio 1',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_regex(
-            self._html_get_attribute_regex('data-channel'), webpage, 'title')
-
-        json_data = json.loads(
-            self._html_search_regex(
-                self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'ext': json_data['codec'],
-            'url': json_data['url']
-        }
-
-    def _html_get_attribute_regex(self, attribute):
-        return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)

From 7105440cecf82aff295df4f32575f6c8b64b3c2d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 13 Feb 2015 15:14:23 +0800
Subject: [PATCH 873/974] [Yam] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/yam.py      | 70 ++++++++++++++++++++++++++++++++
 youtube_dl/utils.py              | 20 +++++++--
 3 files changed, 87 insertions(+), 4 deletions(-)
 create mode 100644 youtube_dl/extractor/yam.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a4fab540b..f749ec333 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -578,6 +578,7 @@ from .yahoo import (
     YahooIE,
     YahooSearchIE,
 )
+from .yam import YamIE
 from .yesjapan import YesJapanIE
 from .ynet import YnetIE
 from .youjizz import YouJizzIE
diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
new file mode 100644
index 000000000..511c8b67c
--- /dev/null
+++ b/youtube_dl/extractor/yam.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import month_by_abbreviation
+import re
+
+
+class YamIE(InfoExtractor):
+    _VALID_URL = r'http://mymedia.yam.com/m/(?P<id>\d+)'
+
+    _TESTS = [{
+        # An audio hosted on Yam
+        'url': 'http://mymedia.yam.com/m/2283921',
+        'md5': 'c011b8e262a52d5473d9c2e3c9963b9c',
+        'info_dict': {
+            'id': '2283921',
+            'ext': 'mp3',
+            'title': '發現 - 趙薇 京華煙雲主題曲',
+            'uploader_id': 'princekt',
+            'upload_date': '20080807',
+            'duration': 313.0,
+        }
+    }, {
+        # An external video hosted on YouTube
+        'url': 'http://mymedia.yam.com/m/3598173',
+        'md5': '0238ceec479c654e8c2f1223755bf3e9',
+        'info_dict': {
+            'id': 'pJ2Deys283c',
+            'ext': 'mp4',
+            'upload_date': '20150202',
+            'uploader': '新莊社大瑜伽社',
+            'description': 'md5:f5cc72f0baf259a70fb731654b0d2eff',
+            'uploader_id': '2323agoy',
+            'title': '外婆的澎湖灣KTV-潘安邦',
+        }
+    }]
+
+    def _real_extract(self, url):
+        media_id = self._match_id(url)
+        page = self._download_webpage(url, media_id)
+
+        # Is it hosted externally on YouTube?
+        youtube_url = self._html_search_regex(
+            r'<embed src="(http://www.youtube.com/[^"]+)"',
+            page, 'YouTube url', default=None)
+        if youtube_url:
+            return self.url_result(youtube_url, 'Youtube')
+
+        api_page = self._download_webpage(
+            'http://mymedia.yam.com/api/a/?pID=' + media_id, media_id)
+        api_result_obj = compat_urlparse.parse_qs(api_page)
+
+        author = self._html_search_regex(
+            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"', page, 'author')
+        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})  ' +
+                         r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
+        upload_date = '%s%02d%02d' % (mobj.group('year'),
+                                      month_by_abbreviation(mobj.group('mon')),
+                                      int(mobj.group('day')))
+
+        return {
+            'id': media_id,
+            'url': api_result_obj['mp3file'][0],
+            'title': self._html_search_meta('description', page),
+            'duration': float(api_result_obj['totaltime'][0]) / 1000.0,
+            'uploader_id': author,
+            'upload_date': upload_date,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 54fa17c38..3eb6bc6d4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -62,6 +62,11 @@ std_headers = {
 }
 
 
+ENGLISH_MONTH_NAMES = [
+    'January', 'February', 'March', 'April', 'May', 'June',
+    'July', 'August', 'September', 'October', 'November', 'December']
+
+
 def preferredencoding():
     """Get preferred encoding.
 
@@ -1185,11 +1190,18 @@ def get_term_width():
 def month_by_name(name):
     """ Return the number of a month by (locale-independently) English name """
 
-    ENGLISH_NAMES = [
-        'January', 'February', 'March', 'April', 'May', 'June',
-        'July', 'August', 'September', 'October', 'November', 'December']
     try:
-        return ENGLISH_NAMES.index(name) + 1
+        return ENGLISH_MONTH_NAMES.index(name) + 1
+    except ValueError:
+        return None
+
+
+def month_by_abbreviation(abbrev):
+    """ Return the number of a month by (locale-independently) English
+        abbreviations """
+
+    try:
+        return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
     except ValueError:
         return None
 

From ae6423d7042de863c303fcbdc1646ea370233741 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Feb 2015 11:36:33 +0100
Subject: [PATCH 874/974] [bambuser] Fix 'uploader_id' extraction (fixes #4944)

---
 youtube_dl/extractor/bambuser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 98e1443ab..c193e66ca 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -50,7 +50,7 @@ class BambuserIE(InfoExtractor):
             'duration': int(info['length']),
             'view_count': int(info['views_total']),
             'uploader': info['username'],
-            'uploader_id': info['uid'],
+            'uploader_id': info['owner']['uid'],
         }
 
 

From 36e7a4ca2e85be8dc16955e28f9c563e01aa9eca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Feb 2015 14:43:50 +0100
Subject: [PATCH 875/974] [test/subtitles] Update checksums

---
 test/test_subtitles.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 6336dd317..3e329438f 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -247,7 +247,7 @@ class TestVimeoSubtitles(BaseTestSubtitles):
     def test_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '26399116d23ae3cf2c087cea94bc43b4')
+        self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
 
     def test_subtitles_lang(self):
         self.DL.params['writesubtitles'] = True
@@ -334,7 +334,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['cs']))
-        self.assertEqual(md5(subtitles['cs']), '9bf52d9549533c32c427e264bf0847d4')
+        self.assertEqual(md5(subtitles['cs']), 'cc3957b2c6dff1db71e5f2e83d467480')
 
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')

From 037e9437e435f32d287354c77f6586d7dc34544b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Feb 2015 20:10:42 +0600
Subject: [PATCH 876/974] [camdemy] Fix _VALID_URL

---
 youtube_dl/extractor/camdemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 5de5879b4..897f3a104 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class CamdemyIE(InfoExtractor):
-    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:www\.)?camdemy\.com/media/(?P<id>\d+)'
     _TESTS = [{
         # single file
         'url': 'http://www.camdemy.com/media/5181/',

From 9347fddbfc81114f85fecc72e8452813ef130fdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 02:04:28 +0600
Subject: [PATCH 877/974] [1tv] Cover arbitraty URLs

---
 youtube_dl/extractor/firsttv.py | 57 ++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 08ceee4ed..6e015ca16 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -8,45 +8,66 @@ from ..utils import int_or_none
 
 
 class FirstTVIE(InfoExtractor):
-    IE_NAME = 'firsttv'
-    IE_DESC = 'Видеоархив - Первый канал'
-    _VALID_URL = r'http://(?:www\.)?1tv\.ru/videoarchive/(?P<id>\d+)'
+    IE_NAME = '1tv'
+    IE_DESC = 'Первый канал'
+    _VALID_URL = r'http://(?:www\.)?1tv\.ru/(?:[^/]+/)+(?P<id>.+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.1tv.ru/videoarchive/73390',
-        'md5': '3de6390cf0cca4a5eae1d1d83895e5ad',
+        'md5': '777f525feeec4806130f4f764bc18a4f',
         'info_dict': {
             'id': '73390',
             'ext': 'mp4',
             'title': 'Олимпийские канатные дороги',
-            'description': 'md5:cc730d2bf4215463e37fff6a1e277b13',
-            'thumbnail': 'http://img1.1tv.ru/imgsize640x360/PR20140210114657.JPG',
+            'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
             'duration': 149,
+            'like_count': int,
+            'dislike_count': int,
         },
         'skip': 'Only works from Russia',
-    }
+    }, {
+        'url': 'http://www.1tv.ru/prj/inprivate/vypusk/35930',
+        'md5': 'a1b6b60d530ebcf8daacf4565762bbaf',
+        'info_dict': {
+            'id': '35930',
+            'ext': 'mp4',
+            'title': 'Наедине со всеми. Людмила Сенчина',
+            'description': 'md5:89553aed1d641416001fe8d450f06cb9',
+            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
+            'duration': 2694,
+        },
+        'skip': 'Only works from Russia',
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id, 'Downloading page')
 
         video_url = self._html_search_regex(
-            r'''(?s)jwplayer\('flashvideoportal_1'\)\.setup\({.*?'file': '([^']+)'.*?}\);''', webpage, 'video URL')
+            r'''(?s)(?:jwplayer\('flashvideoportal_1'\)\.setup\({|var\s+playlistObj\s*=).*?'file'\s*:\s*'([^']+)'.*?}\);''',
+            webpage, 'video URL')
 
         title = self._html_search_regex(
-            r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>', webpage, 'title')
+            [r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>',
+             r"'title'\s*:\s*'([^']+)'"], webpage, 'title')
         description = self._html_search_regex(
-            r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>', webpage, 'description', fatal=False)
+            r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>',
+            webpage, 'description', default=None) or self._html_search_meta(
+                'description', webpage, 'description')
 
         thumbnail = self._og_search_thumbnail(webpage)
-        duration = self._og_search_property('video:duration', webpage, 'video duration', fatal=False)
+        duration = self._og_search_property(
+            'video:duration', webpage,
+            'video duration', fatal=False)
 
-        like_count = self._html_search_regex(r'title="Понравилось".*?/></label> \[(\d+)\]',
-                                             webpage, 'like count', fatal=False)
-        dislike_count = self._html_search_regex(r'title="Не понравилось".*?/></label> \[(\d+)\]',
-                                                webpage, 'dislike count', fatal=False)
+        like_count = self._html_search_regex(
+            r'title="Понравилось".*?/></label> \[(\d+)\]',
+            webpage, 'like count', default=None)
+        dislike_count = self._html_search_regex(
+            r'title="Не понравилось".*?/></label> \[(\d+)\]',
+            webpage, 'dislike count', default=None)
 
         return {
             'id': video_id,

From cffcbc02de504d84e1c2677bb525c00b03e04f53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Feb 2015 22:25:34 +0100
Subject: [PATCH 878/974] [postprocessor/ffmpeg] Don't let ffmpeg read from
 stdin (fixes #4945)

If you run 'while read aurl ; do youtube-dl --extract-audio "${aurl}"; done < path_to_batch_file'  (batch_file contains one url per line) each call to youtube-dl consumed some characters and 'read' would assing to 'aurl' a non valid url, something like 'tube.com/watch?v=<id>'.
---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 01d25f760..5238ce534 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -95,7 +95,7 @@ class FFmpegPostProcessor(PostProcessor):
 
         if self._downloader.params.get('verbose', False):
             self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd))
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
         stdout, stderr = p.communicate()
         if p.returncode != 0:
             stderr = stderr.decode('utf-8', 'replace')
@@ -134,7 +134,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 encodeFilename(self._probe_executable, True),
                 encodeArgument('-show_streams'),
                 encodeFilename(self._ffmpeg_filename_argument(path), True)]
-            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
+            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE, stdin=subprocess.PIPE)
             output = handle.communicate()[0]
             if handle.wait() != 0:
                 return None

From d9aa2b784d914ae99c7d9bbaf83d06f1b9dbd04e Mon Sep 17 00:00:00 2001
From: Ryan Schmidt <github@ryandesign.com>
Date: Sat, 14 Feb 2015 04:10:23 -0600
Subject: [PATCH 879/974] Support NBC Nightly News broadcasts

---
 youtube_dl/extractor/nbc.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index f840f6532..3e3de9e2d 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -54,7 +54,7 @@ class NBCIE(InfoExtractor):
 class NBCNewsIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://www\.nbcnews\.com/
         ((video/.+?/(?P<id>\d+))|
-        (feature/[^/]+/(?P<title>.+)))
+        ((?P<program>feature|nightly-news)/[^/]+/(?P<title>.+)))
         '''
 
     _TESTS = [
@@ -89,6 +89,16 @@ class NBCNewsIE(InfoExtractor):
                 'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
             },
         },
+        {
+            'url': 'http://www.nbcnews.com/nightly-news/video/nightly-news-with-brian-williams-full-broadcast-february-4-394064451844',
+            'md5': 'b5dda8cddd8650baa0dcb616dd2cf60d',
+            'info_dict': {
+                'id': 'sekXqyTVnmN3',
+                'ext': 'mp4',
+                'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
+                'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -107,12 +117,19 @@ class NBCNewsIE(InfoExtractor):
                 'thumbnail': find_xpath_attr(info, 'media', 'type', 'thumbnail').text,
             }
         else:
-            # "feature" pages use theplatform.com
+            # "feature" and "nightly-news" pages use theplatform.com
             title = mobj.group('title')
             webpage = self._download_webpage(url, title)
-            bootstrap_json = self._search_regex(
-                r'var bootstrapJson = ({.+})\s*$', webpage, 'bootstrap json',
-                flags=re.MULTILINE)
+            program = mobj.group('program')
+            if program == 'feature':
+                bootstrap_json = self._search_regex(
+                    r'var bootstrapJson = ({.+})\s*$', webpage, 'bootstrap json',
+                    flags=re.MULTILINE)
+            else:
+                # nightly-news
+                bootstrap_json = self._search_regex(
+                    r'var playlistData = ({.+});\s*$', webpage, 'playlist data',
+                    flags=re.MULTILINE)
             bootstrap = json.loads(bootstrap_json)
             info = bootstrap['results'][0]['video']
             mpxid = info['mpxId']

From a4f3d779db13ec3c8bda67e897de8cd849a7f811 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 17:42:12 +0600
Subject: [PATCH 880/974] [nbcnews] Simplify

---
 youtube_dl/extractor/nbc.py | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 3e3de9e2d..52e8595ea 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -52,9 +52,9 @@ class NBCIE(InfoExtractor):
 
 
 class NBCNewsIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://www\.nbcnews\.com/
-        ((video/.+?/(?P<id>\d+))|
-        ((?P<program>feature|nightly-news)/[^/]+/(?P<title>.+)))
+    _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
+        (?:video/.+?/(?P<id>\d+)|
+        (?:feature|nightly-news)/[^/]+/(?P<title>.+))
         '''
 
     _TESTS = [
@@ -120,17 +120,10 @@ class NBCNewsIE(InfoExtractor):
             # "feature" and "nightly-news" pages use theplatform.com
             title = mobj.group('title')
             webpage = self._download_webpage(url, title)
-            program = mobj.group('program')
-            if program == 'feature':
-                bootstrap_json = self._search_regex(
-                    r'var bootstrapJson = ({.+})\s*$', webpage, 'bootstrap json',
-                    flags=re.MULTILINE)
-            else:
-                # nightly-news
-                bootstrap_json = self._search_regex(
-                    r'var playlistData = ({.+});\s*$', webpage, 'playlist data',
-                    flags=re.MULTILINE)
-            bootstrap = json.loads(bootstrap_json)
+            bootstrap_json = self._search_regex(
+                r'var\s+(?:bootstrapJson|playlistData)\s*=\s*({.+});?\s*$',
+                webpage, 'bootstrap json', flags=re.MULTILINE)
+            bootstrap = self._parse_json(bootstrap_json, video_id)
             info = bootstrap['results'][0]['video']
             mpxid = info['mpxId']
 

From 3215c50f25bcf01a037e066747d7d60bd274ae71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 17:44:24 +0600
Subject: [PATCH 881/974] Credit @ryandesign for nbcnews nightly news (#4948)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 3d6985ab6..47f12a9ee 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -110,3 +110,4 @@ Shaya Goldberg
 Paul Hartmann
 Frans de Jonge
 Robin de Rooij
+Ryan Schmidt

From ae1580d790ecca2b6ad132da26bc834e4f9873f3 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Sat, 14 Feb 2015 13:29:44 +0100
Subject: [PATCH 882/974] [sunporno] fix extraction

---
 youtube_dl/extractor/sunporno.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 8a333f1d2..f2a1d6550 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -52,7 +52,7 @@ class SunPornoIE(InfoExtractor):
 
         formats = []
         quality = qualities(['mp4', 'flv'])
-        for video_url in re.findall(r'<source src="([^"]+)"', webpage):
+        for video_url in re.findall(r'<video src="([^"]+)"', webpage):
             video_ext = determine_ext(video_url)
             formats.append({
                 'url': video_url,

From 68f2d273bfeb7aa10eba68b6c62cc0502a948c72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 18:33:52 +0600
Subject: [PATCH 883/974] [sunporno] Keep old video regex just in case

---
 youtube_dl/extractor/sunporno.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index f2a1d6550..854d01bee 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -52,7 +52,7 @@ class SunPornoIE(InfoExtractor):
 
         formats = []
         quality = qualities(['mp4', 'flv'])
-        for video_url in re.findall(r'<video src="([^"]+)"', webpage):
+        for video_url in re.findall(r'<(?:source|video) src="([^"]+)"', webpage):
             video_ext = determine_ext(video_url)
             formats.append({
                 'url': video_url,

From 10e3c4c2215a7d65391d23efd19aab96197cdcf8 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Sat, 14 Feb 2015 13:40:35 +0100
Subject: [PATCH 884/974] [drtuber] fix extraction

---
 youtube_dl/extractor/drtuber.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index ca274dff6..2f06e64cc 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -36,7 +36,7 @@ class DrTuberIE(InfoExtractor):
             r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*Free', webpage, 'title')
+            r'<title>([^<]+) - \d+', webpage, 'title')
 
         thumbnail = self._html_search_regex(
             r'poster="([^"]+)"',

From 52e1d0ccc415377bd9f7d862f645c87909705ace Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Sat, 14 Feb 2015 13:42:42 +0100
Subject: [PATCH 885/974] [beeg] fix test

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 4e79fea8f..b38057f2f 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -9,7 +9,7 @@ class BeegIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
     _TEST = {
         'url': 'http://beeg.com/5416503',
-        'md5': '634526ae978711f6b748fe0dd6c11f57',
+        'md5': '1bff67111adb785c51d1b42959ec10e5',
         'info_dict': {
             'id': '5416503',
             'ext': 'mp4',

From 76d1466b08d4e3cebc40a7fb7dae957c3b34b1eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 18:50:13 +0600
Subject: [PATCH 886/974] [drtuber] Add one more title regex

---
 youtube_dl/extractor/drtuber.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index 2f06e64cc..37c5c181f 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -15,7 +15,7 @@ class DrTuberIE(InfoExtractor):
             'id': '1740434',
             'display_id': 'hot-perky-blonde-naked-golf',
             'ext': 'mp4',
-            'title': 'Hot Perky Blonde Naked Golf',
+            'title': 'hot perky blonde naked golf',
             'like_count': int,
             'dislike_count': int,
             'comment_count': int,
@@ -36,7 +36,8 @@ class DrTuberIE(InfoExtractor):
             r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._html_search_regex(
-            r'<title>([^<]+) - \d+', webpage, 'title')
+            [r'class="hd_title" style="[^"]+">([^<]+)</h1>', r'<title>([^<]+) - \d+'],
+            webpage, 'title')
 
         thumbnail = self._html_search_regex(
             r'poster="([^"]+)"',

From a294bce82f84482091fa6f44ce913a7a86fe8be8 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 14 Feb 2015 17:48:04 +0200
Subject: [PATCH 887/974] [streamcz] Fix extraction (Closes #4940)

---
 youtube_dl/extractor/streamcz.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index c3ceb5f76..e92b93285 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -1,14 +1,30 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import hashlib
+import time
+
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+)
 from ..utils import (
     int_or_none,
 )
 
 
+def _get_api_key(api_path):
+    if api_path.endswith('?'):
+        api_path = api_path[:-1]
+
+    api_key = 'fb5f58a820353bd7095de526253c14fd'
+    a = '{0:}{1:}{2:}'.format(api_key, api_path, int(round(time.time() / 24 / 3600)))
+    return hashlib.md5(a.encode('ascii')).hexdigest()
+
+
 class StreamCZIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?stream\.cz/.+/(?P<id>[0-9]+)'
+    _API_URL = 'http://www.stream.cz/API'
 
     _TESTS = [{
         'url': 'http://www.stream.cz/peklonataliri/765767-ecka-pro-deti',
@@ -36,8 +52,11 @@ class StreamCZIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        data = self._download_json(
-            'http://www.stream.cz/API/episode/%s' % video_id, video_id)
+        api_path = '/episode/%s' % video_id
+
+        req = compat_urllib_request.Request(self._API_URL + api_path)
+        req.add_header('Api-Password', _get_api_key(api_path))
+        data = self._download_json(req, video_id)
 
         formats = []
         for quality, video in enumerate(data['video_qualities']):

From b0ab0fac49057ea0df3db75fb407af92deaaa453 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Feb 2015 22:18:09 +0100
Subject: [PATCH 888/974] Remove unused imports

---
 youtube_dl/extractor/firsttv.py | 2 --
 youtube_dl/extractor/nbc.py     | 1 -
 2 files changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 6e015ca16..510d4b108 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import int_or_none
 
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 52e8595ea..89a2845fe 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..compat import (

From 6ca7732d5e792f02be030d4a9c9d101fff2f4079 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Feb 2015 22:20:24 +0100
Subject: [PATCH 889/974] [extractor/common] Fix link to external documentation

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 48742189a..c784eedb9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -665,7 +665,7 @@ class InfoExtractor(object):
         return RATING_TABLE.get(rating.lower(), None)
 
     def _family_friendly_search(self, html):
-        # See http://schema.org/VideoObj
+        # See http://schema.org/VideoObject
         family_friendly = self._html_search_meta('isFamilyFriendly', html)
 
         if not family_friendly:

From 9fb2f1cd6d14dafca7a2d6cd74b0394b0f24afae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Feb 2015 04:56:12 +0600
Subject: [PATCH 890/974] [theplatform] Add URL sign capability

---
 youtube_dl/extractor/theplatform.py | 42 ++++++++++++++++++++++++++---
 1 file changed, 38 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 110ed976d..1579822f2 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -2,6 +2,11 @@ from __future__ import unicode_literals
 
 import re
 import json
+import time
+import hmac
+import binascii
+import hashlib
+
 
 from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
@@ -11,6 +16,7 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     xpath_with_ns,
+    unsmuggle_url,
 )
 
 _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'})
@@ -18,7 +24,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language
 
 class ThePlatformIE(SubtitlesInfoExtractor):
     _VALID_URL = r'''(?x)
-        (?:https?://(?:link|player)\.theplatform\.com/[sp]/[^/]+/
+        (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
            (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
@@ -38,9 +44,33 @@ class ThePlatformIE(SubtitlesInfoExtractor):
         },
     }
 
+    @staticmethod
+    def _sign_url(url, sig_key, sig_secret, life=600, include_qs=False):
+        flags = '10' if include_qs else '00'
+        expiration_date = '%x' % (int(time.time()) + life)
+
+        def str_to_hex(str):
+            return binascii.b2a_hex(str.encode('ascii')).decode('ascii')
+
+        def hex_to_str(hex):
+            return binascii.a2b_hex(hex)
+
+        relative_path = url.split('http://link.theplatform.com/s/')[1].split('?')[0]
+        clear_text = hex_to_str(flags + expiration_date + str_to_hex(relative_path))
+        checksum = hmac.new(sig_key.encode('ascii'), clear_text, hashlib.sha1).hexdigest()
+        sig = flags + expiration_date + checksum + str_to_hex(sig_secret)
+        return '%s&sig=%s' % (url, sig)
+
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         mobj = re.match(self._VALID_URL, url)
+        provider_id = mobj.group('provider_id')
         video_id = mobj.group('id')
+
+        if not provider_id:
+            provider_id = 'dJ5BDC'
+
         if mobj.group('config'):
             config_url = url + '&form=json'
             config_url = config_url.replace('swf/', 'config/')
@@ -48,8 +78,12 @@ class ThePlatformIE(SubtitlesInfoExtractor):
             config = self._download_json(config_url, video_id, 'Downloading config')
             smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
         else:
-            smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?'
-                        'format=smil&mbr=true'.format(video_id))
+            smil_url = ('http://link.theplatform.com/s/{0}/{1}/meta.smil?'
+                        'format=smil&mbr=true'.format(provider_id, video_id))
+
+        sig = smuggled_data.get('sig')
+        if sig:
+            smil_url = self._sign_url(smil_url, sig['key'], sig['secret'])
 
         meta = self._download_xml(smil_url, video_id)
         try:
@@ -62,7 +96,7 @@ class ThePlatformIE(SubtitlesInfoExtractor):
         else:
             raise ExtractorError(error_msg, expected=True)
 
-        info_url = 'http://link.theplatform.com/s/dJ5BDC/{0}?format=preview'.format(video_id)
+        info_url = 'http://link.theplatform.com/s/{0}/{1}?format=preview'.format(provider_id, video_id)
         info_json = self._download_webpage(info_url, video_id)
         info = json.loads(info_json)
 

From b9c7a97318f4ca68cacf0ff5395c0d4a6f74b525 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Feb 2015 04:57:52 +0600
Subject: [PATCH 891/974] [history] Add extractor (Closes #4934)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/history.py  | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 youtube_dl/extractor/history.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a4fab540b..13292073c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -189,6 +189,7 @@ from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
 from .historicfilms import HistoricFilmsIE
+from .history import HistoryIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hostingbulk import HostingBulkIE
diff --git a/youtube_dl/extractor/history.py b/youtube_dl/extractor/history.py
new file mode 100644
index 000000000..f86164afe
--- /dev/null
+++ b/youtube_dl/extractor/history.py
@@ -0,0 +1,31 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class HistoryIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?history\.com/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
+
+    _TESTS = [{
+        'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
+        'md5': '6fe632d033c92aa10b8d4a9be047a7c5',
+        'info_dict': {
+            'id': 'bLx5Dv5Aka1G',
+            'ext': 'mp4',
+            'title': "Bet You Didn't Know: Valentine's Day",
+            'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
+        },
+        'add_ie': ['ThePlatform'],
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._search_regex(
+            r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
+            webpage, 'video url')
+
+        return self.url_result(smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}))

From f813928e4b669627007b772c9b150eed135b18be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Feb 2015 16:32:38 +0600
Subject: [PATCH 892/974] [bbccouk] Fix fallback to legacy playlist

---
 youtube_dl/extractor/bbccouk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 126c8824c..f23e39545 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -273,7 +273,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
                     formats, subtitles = self._download_media_selector(programme_id)
                 return programme_id, title, description, duration, formats, subtitles
         except ExtractorError as ee:
-            if not isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
+            if not (isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404):
                 raise
 
         # fallback to legacy playlist

From 8fb474fb17a64ff2aa9f6315ebbc99ae7938c4e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Feb 2015 14:59:00 +0100
Subject: [PATCH 893/974] [test/subtitles] Fix some tests

The checksym for the CeskaTelevize subtitles has changed again, so we just test that it has a reasonable length.
---
 test/test_subtitles.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 3e329438f..bcc69a778 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -138,7 +138,7 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles.keys()), 5)
+        self.assertTrue(len(subtitles.keys()) >= 6)
 
     def test_list_subtitles(self):
         self.DL.expect_warning('Automatic Captions not supported by this server')
@@ -334,7 +334,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['cs']))
-        self.assertEqual(md5(subtitles['cs']), 'cc3957b2c6dff1db71e5f2e83d467480')
+        self.assertTrue(len(subtitles['cs']) > 20000)
 
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')

From 73fac4e9119bc53d49e646c38529f1b6d8de734d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 13 Feb 2015 11:14:01 +0100
Subject: [PATCH 894/974] [ffmpeg] Add --ffmpeg-location

---
 youtube_dl/YoutubeDL.py            |  4 +-
 youtube_dl/__init__.py             |  1 +
 youtube_dl/downloader/hls.py       |  7 +--
 youtube_dl/options.py              |  4 ++
 youtube_dl/postprocessor/ffmpeg.py | 95 ++++++++++++++++++++++--------
 5 files changed, 80 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 13d18e25e..dbb26272d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1298,7 +1298,7 @@ class YoutubeDL(object):
                     downloaded = []
                     success = True
                     merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
-                    if not merger._executable:
+                    if not merger.available():
                         postprocessors = []
                         self.report_warning('You have requested multiple '
                                             'formats but ffmpeg or avconv are not installed.'
@@ -1647,7 +1647,7 @@ class YoutubeDL(object):
         self._write_string('[debug] Python version %s - %s\n' % (
             platform.python_version(), platform_name()))
 
-        exe_versions = FFmpegPostProcessor.get_versions()
+        exe_versions = FFmpegPostProcessor.get_versions(self)
         exe_versions['rtmpdump'] = rtmpdump_version()
         exe_str = ', '.join(
             '%s %s' % (exe, v)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ed22f169f..108fb3c7a 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -350,6 +350,7 @@ def _real_main(argv=None):
         'xattr_set_filesize': opts.xattr_set_filesize,
         'match_filter': match_filter,
         'no_color': opts.no_color,
+        'ffmpeg_location': opts.ffmpeg_location,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index e527ee425..8be4f4249 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -23,15 +23,14 @@ class HlsFD(FileDownloader):
         tmpfilename = self.temp_name(filename)
 
         ffpp = FFmpegPostProcessor(downloader=self)
-        program = ffpp._executable
-        if program is None:
+        if not ffpp.available:
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()
 
         args = [
             encodeArgument(opt)
-            for opt in (program, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
+            for opt in (ffpp.executable, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
         args.append(encodeFilename(tmpfilename, True))
 
         retval = subprocess.call(args)
@@ -48,7 +47,7 @@ class HlsFD(FileDownloader):
             return True
         else:
             self.to_stderr('\n')
-            self.report_error('%s exited with code %d' % (program, retval))
+            self.report_error('%s exited with code %d' % (ffpp.basename, retval))
             return False
 
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 873432bee..ba35399cf 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -735,6 +735,10 @@ def parseOpts(overrideArguments=None):
         '--prefer-ffmpeg',
         action='store_true', dest='prefer_ffmpeg',
         help='Prefer ffmpeg over avconv for running the postprocessors')
+    postproc.add_option(
+        '--ffmpeg-location', '--avconv-location', metavar='PATH',
+        dest='ffmpeg_location',
+        help='Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.')
     postproc.add_option(
         '--exec',
         metavar='CMD', dest='exec_cmd',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 01d25f760..504a71193 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -30,54 +30,97 @@ class FFmpegPostProcessorError(PostProcessingError):
 class FFmpegPostProcessor(PostProcessor):
     def __init__(self, downloader=None, deletetempfiles=False):
         PostProcessor.__init__(self, downloader)
-        self._versions = self.get_versions()
         self._deletetempfiles = deletetempfiles
+        self._determine_executables()
 
     def check_version(self):
-        if not self._executable:
+        if not self.available():
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
         required_version = '10-0' if self._uses_avconv() else '1.0'
         if is_outdated_version(
-                self._versions[self._executable], required_version):
+                self._versions[self.basename], required_version):
             warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
-                self._executable, self._executable, required_version)
+                self.basename, self.basename, required_version)
             if self._downloader:
                 self._downloader.report_warning(warning)
 
     @staticmethod
-    def get_versions():
+    def get_versions(downloader=None):
+        return FFmpegPostProcessor(downloader)._versions
+
+    def _determine_executables(self):
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-        return dict((p, get_exe_version(p, args=['-version'])) for p in programs)
+        prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', False)
 
-    @property
-    def available(self):
-        return self._executable is not None
+        self.basename = None
+        self.probe_basename = None
 
-    @property
-    def _executable(self):
-        if self._downloader.params.get('prefer_ffmpeg', False):
+        self._paths = None
+        self._versions = None
+        if self._downloader:
+            location = self._downloader.params.get('ffmpeg_location')
+            if location is not None:
+                if not os.path.exists(location):
+                    self._downloader.report_warning(
+                        'ffmpeg-location %s does not exist! '
+                        'Continuing without avconv/ffmpeg.' % (location))
+                    self._versions = {}
+                    return
+                elif not os.path.isdir(location):
+                    basename = os.path.splitext(os.path.basename(location))[0]
+                    if basename not in programs:
+                        self._downloader.report_warning(
+                            'Cannot identify executable %s, its basename should be one of %s. '
+                            'Continuing without avconv/ffmpeg.' %
+                            (location, ', '.join(programs)))
+                        self._versions = {}
+                        return None
+                    location = os.path.dirname(os.path.abspath(location))
+                    if basename in ('ffmpeg', 'ffprobe'):
+                        prefer_ffmpeg = True
+
+                self._paths = dict(
+                    (p, os.path.join(location, p)) for p in programs)
+                self._versions = dict(
+                    (p, get_exe_version(self._paths[p], args=['-version']))
+                    for p in programs)
+        if self._versions is None:
+            self._versions = dict(
+                (p, get_exe_version(p, args=['-version'])) for p in programs)
+            self._paths = dict((p, p) for p in programs)
+
+        if prefer_ffmpeg:
             prefs = ('ffmpeg', 'avconv')
         else:
             prefs = ('avconv', 'ffmpeg')
         for p in prefs:
             if self._versions[p]:
-                return p
-        return None
+                self.basename = p
+                break
 
-    @property
-    def _probe_executable(self):
-        if self._downloader.params.get('prefer_ffmpeg', False):
+        if prefer_ffmpeg:
             prefs = ('ffprobe', 'avprobe')
         else:
             prefs = ('avprobe', 'ffprobe')
         for p in prefs:
             if self._versions[p]:
-                return p
-        return None
+                self.probe_basename = p
+                break
+
+    def available(self):
+        return self.basename is not None
 
     def _uses_avconv(self):
-        return self._executable == 'avconv'
+        return self.basename == 'avconv'
+
+    @property
+    def executable(self):
+        return self._paths[self.basename]
+
+    @property
+    def probe_executable(self):
+        return self._paths[self.probe_basename]
 
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
         self.check_version()
@@ -88,7 +131,7 @@ class FFmpegPostProcessor(PostProcessor):
         files_cmd = []
         for path in input_paths:
             files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
-        cmd = ([encodeFilename(self._executable, True), encodeArgument('-y')] +
+        cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] +
                files_cmd +
                [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
@@ -127,13 +170,15 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
     def get_audio_codec(self, path):
 
-        if not self._probe_executable:
+        if not self.probe_executable:
             raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
         try:
             cmd = [
-                encodeFilename(self._probe_executable, True),
+                encodeFilename(self.probe_executable, True),
                 encodeArgument('-show_streams'),
                 encodeFilename(self._ffmpeg_filename_argument(path), True)]
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] ffprobe command line: %s' % shell_quote(cmd))
             handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
             output = handle.communicate()[0]
             if handle.wait() != 0:
@@ -223,14 +268,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
                 self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
             else:
-                self._downloader.to_screen('[' + self._executable + '] Destination: ' + new_path)
+                self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
                 self.run_ffmpeg(path, new_path, acodec, more_opts)
         except:
             etype, e, tb = sys.exc_info()
             if isinstance(e, AudioConversionError):
                 msg = 'audio conversion failed: ' + e.msg
             else:
-                msg = 'error running ' + self._executable
+                msg = 'error running ' + self.basename
             raise PostProcessingError(msg)
 
         # Try to update the date time for extracted audio file.

From 03cd72b0070c430e95b7e9d612afcad026690153 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 04:37:55 +0100
Subject: [PATCH 895/974] [extractor/common] Move up filesize

filesize and tbr should correlate, so it doesn't make sense to treat them differently.
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c784eedb9..e74b7bf25 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -729,6 +729,7 @@ class InfoExtractor(object):
                 f.get('language_preference') if f.get('language_preference') is not None else -1,
                 f.get('quality') if f.get('quality') is not None else -1,
                 f.get('tbr') if f.get('tbr') is not None else -1,
+                f.get('filesize') if f.get('filesize') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
@@ -736,7 +737,6 @@ class InfoExtractor(object):
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('fps') if f.get('fps') is not None else -1,
-                f.get('filesize') if f.get('filesize') is not None else -1,
                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
                 f.get('source_preference') if f.get('source_preference') is not None else -1,
                 f.get('format_id'),

From 85e80f71cddafe956886968d0eb60eff0aa7432c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 04:50:57 +0100
Subject: [PATCH 896/974] [yam] Allow faults in optional fields (#4943)

---
 youtube_dl/extractor/yam.py | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 511c8b67c..b294767c5 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -1,10 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_urlparse
-from ..utils import month_by_abbreviation
-import re
+from ..utils import (
+    float_or_none,
+    month_by_abbreviation,
+)
 
 
 class YamIE(InfoExtractor):
@@ -38,8 +42,8 @@ class YamIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        media_id = self._match_id(url)
-        page = self._download_webpage(url, media_id)
+        video_id = self._match_id(url)
+        page = self._download_webpage(url, video_id)
 
         # Is it hosted externally on YouTube?
         youtube_url = self._html_search_regex(
@@ -49,22 +53,29 @@ class YamIE(InfoExtractor):
             return self.url_result(youtube_url, 'Youtube')
 
         api_page = self._download_webpage(
-            'http://mymedia.yam.com/api/a/?pID=' + media_id, media_id)
+            'http://mymedia.yam.com/api/a/?pID=' + video_id, video_id,
+            note='Downloading API page')
         api_result_obj = compat_urlparse.parse_qs(api_page)
 
-        author = self._html_search_regex(
-            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"', page, 'author')
+        uploader_id = self._html_search_regex(
+            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"',
+            page, 'uploader id', fatal=False)
         mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})  ' +
                          r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
-        upload_date = '%s%02d%02d' % (mobj.group('year'),
-                                      month_by_abbreviation(mobj.group('mon')),
-                                      int(mobj.group('day')))
+        if mobj:
+            upload_date = '%s%02d%02d' % (
+                mobj.group('year'),
+                month_by_abbreviation(mobj.group('mon')),
+                int(mobj.group('day')))
+        else:
+            upload_date = None
+        duration = float_or_none(api_result_obj['totaltime'][0], scale=1000)
 
         return {
-            'id': media_id,
+            'id': video_id,
             'url': api_result_obj['mp3file'][0],
             'title': self._html_search_meta('description', page),
-            'duration': float(api_result_obj['totaltime'][0]) / 1000.0,
-            'uploader_id': author,
+            'duration': duration,
+            'uploader_id': uploader_id,
             'upload_date': upload_date,
         }

From a5681804415a9b8d011cd238b7d61e6eb4ec7c4b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 04:51:20 +0100
Subject: [PATCH 897/974] release 2015.02.16

---
 README.md              | 3 +++
 docs/supportedsites.md | 8 +++++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 731cea1e1..d68896202 100644
--- a/README.md
+++ b/README.md
@@ -397,6 +397,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
                                      postprocessors
+    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary;
+                                     either the path to the binary or its
+                                     containing directory.
     --exec CMD                       Execute a command on the file after
                                      downloading, similar to find's -exec
                                      syntax. Example: --exec 'adb push {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8bce8fede..894fa4323 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1,4 +1,5 @@
 # Supported sites
+ - **1tv**: Первый канал
  - **1up.com**
  - **220.ro**
  - **24video**
@@ -60,6 +61,8 @@
  - **Brightcove**
  - **BuzzFeed**
  - **BYUtv**
+ - **Camdemy**
+ - **CamdemyFolder**
  - **Canal13cl**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
@@ -134,7 +137,6 @@
  - **fernsehkritik.tv:postecke**
  - **Firedrive**
  - **Firstpost**
- - **firsttv**: Видеоархив - Первый канал
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **Foxgay**
@@ -174,6 +176,7 @@
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
  - **HistoricFilms**
+ - **History**
  - **hitbox**
  - **hitbox:live**
  - **HornBunny**
@@ -287,6 +290,8 @@
  - **nowvideo**: NowVideo
  - **npo.nl**
  - **npo.nl:live**
+ - **npo.nl:radio**
+ - **npo.nl:radio:fragment**
  - **NRK**
  - **NRKTV**
  - **ntv.ru**
@@ -527,6 +532,7 @@
  - **XVideos**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
+ - **Yam**
  - **YesJapan**
  - **Ynet**
  - **YouJizz**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 492ddf2ea..e58c59969 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.11'
+__version__ = '2015.02.16'

From 59b8ab583491410f4f365b1cb268a4360d7be877 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 15:45:01 +0100
Subject: [PATCH 898/974] [rtlnl|generic] Add support for rtl.nl embeds (Fixes
 #4959)

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/generic.py  | 17 ++++++++-
 youtube_dl/extractor/rtlnl.py    | 63 ++++++++++++++++++++++++--------
 3 files changed, 65 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5b776a7a1..7d0909b5d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -371,7 +371,7 @@ from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
 from .rte import RteIE
-from .rtlnl import RtlXlIE
+from .rtlnl import RtlNlIE
 from .rtlnow import RTLnowIE
 from .rtl2 import RTL2IE
 from .rtp import RTPIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f4500e931..c4ba25a96 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -537,6 +537,15 @@ class GenericIE(InfoExtractor):
                 'uploader_id': 'NationalArchives08',
                 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
             },
+        },
+        # rtl.nl embed
+        {
+            'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
+            'playlist_mincount': 5,
+            'info_dict': {
+                'id': 'aanslagen-kopenhagen',
+                'title': 'Aanslagen Kopenhagen | RTL Nieuws',
+            }
         }
     ]
 
@@ -782,6 +791,13 @@ class GenericIE(InfoExtractor):
                 'entries': entries,
             }
 
+        # Look for embedded rtl.nl player
+        matches = re.findall(
+            r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+video_embed[^"]+)"',
+            webpage)
+        if matches:
+            return _playlist_from_matches(matches, ie='RtlNl')
+
         # Look for embedded (iframe) Vimeo player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
@@ -789,7 +805,6 @@ class GenericIE(InfoExtractor):
             player_url = unescapeHTML(mobj.group('url'))
             surl = smuggle_url(player_url, {'Referer': url})
             return self.url_result(surl)
-
         # Look for embedded (swf embed) Vimeo player
         mobj = re.search(
             r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index a3ca79f2c..cfce4550a 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -1,16 +1,25 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import parse_duration
+from ..utils import (
+    int_or_none,
+    parse_duration,
+)
 
 
-class RtlXlIE(InfoExtractor):
-    IE_NAME = 'rtlxl.nl'
-    _VALID_URL = r'https?://(www\.)?rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)'
+class RtlNlIE(InfoExtractor):
+    IE_NAME = 'rtl.nl'
+    IE_DESC = 'rtl.nl and rtlxl.nl'
+    _VALID_URL = r'''(?x)
+        https?://(www\.)?
+        (?:
+            rtlxl\.nl/\#!/[^/]+/|
+            rtl\.nl/system/videoplayer/[^?#]+?/video_embed\.html\#uuid=
+        )
+        (?P<id>[0-9a-f-]+)'''
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677',
         'md5': 'cc16baa36a6c169391f0764fa6b16654',
         'info_dict': {
@@ -22,21 +31,30 @@ class RtlXlIE(InfoExtractor):
             'upload_date': '20140814',
             'duration': 576.880,
         },
-    }
+    }, {
+        'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed/autoplay=false',
+        'md5': 'dea7474214af1271d91ef332fb8be7ea',
+        'info_dict': {
+            'id': '84ae5571-ac25-4225-ae0c-ef8d9efb2aed',
+            'ext': 'mp4',
+            'timestamp': 1424039400,
+            'title': 'RTL Nieuws - Nieuwe beelden Kopenhagen: chaos direct na aanslag',
+            'thumbnail': 're:^https?://screenshots\.rtl\.nl/system/thumb/sz=[0-9]+x[0-9]+/uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed$',
+            'upload_date': '20150215',
+            'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
+        }
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        uuid = mobj.group('uuid')
-
+        uuid = self._match_id(url)
         info = self._download_json(
             'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
             uuid)
 
         material = info['material'][0]
-        episode_info = info['episodes'][0]
-
         progname = info['abstracts'][0]['name']
         subtitle = material['title'] or info['episodes'][0]['name']
+        description = material.get('synopsis') or info['episodes'][0]['synopsis']
 
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
         videopath = material['videopath'].replace('.f4m', '.m3u8')
@@ -58,14 +76,29 @@ class RtlXlIE(InfoExtractor):
                 'quality': 0,
             }
         ])
-
         self._sort_formats(formats)
 
+        thumbnails = []
+        meta = info.get('meta', {})
+        for p in ('poster_base_url', '"thumb_base_url"'):
+            if not meta.get(p):
+                continue
+
+            thumbnails.append({
+                'url': self._proto_relative_url(meta[p] + uuid),
+                'width': int_or_none(self._search_regex(
+                    r'/sz=([0-9]+)', meta[p], 'thumbnail width', fatal=False)),
+                'height': int_or_none(self._search_regex(
+                    r'/sz=[0-9]+x([0-9]+)',
+                    meta[p], 'thumbnail height', fatal=False))
+            })
+
         return {
             'id': uuid,
             'title': '%s - %s' % (progname, subtitle),
             'formats': formats,
             'timestamp': material['original_date'],
-            'description': episode_info['synopsis'],
+            'description': description,
             'duration': parse_duration(material.get('duration')),
+            'thumbnails': thumbnails,
         }

From 1db5fbcfe3ef95654aa642ebe483fb070852cb49 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 15:47:13 +0100
Subject: [PATCH 899/974] release 2015.02.16.1

---
 docs/supportedsites.md | 2 +-
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 894fa4323..4bb68fdc5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -338,9 +338,9 @@
  - **Roxwel**
  - **RTBF**
  - **Rte**
+ - **rtl.nl**: rtl.nl and rtlxl.nl
  - **RTL2**
  - **RTLnow**
- - **rtlxl.nl**
  - **RTP**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e58c59969..bbe0e53b5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.16'
+__version__ = '2015.02.16.1'

From 85729c51afad484ef784faf5d82bad8acab77d5e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 12:09:12 +0100
Subject: [PATCH 900/974] [downloader] Add --hls-prefer-native to use the
 native HLS downloader (#4966)

---
 youtube_dl/YoutubeDL.py           | 5 ++++-
 youtube_dl/__init__.py            | 1 +
 youtube_dl/downloader/__init__.py | 3 +++
 youtube_dl/options.py             | 4 ++++
 4 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dbb26272d..ea2435e0a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -225,7 +225,6 @@ class YoutubeDL(object):
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
-    external_downloader:  Executable of the external downloader to call.
     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
     match_filter:      A function that gets called with the info_dict of
@@ -235,6 +234,10 @@ class YoutubeDL(object):
                        match_filter_func in utils.py is one example for this.
     no_color:          Do not emit color codes in output.
 
+    The following options determine which downloader is picked:
+    external_downloader: Executable of the external downloader to call.
+                       None or unset for standard (built-in) downloader.
+    hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 108fb3c7a..eac2a26ec 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -351,6 +351,7 @@ def _real_main(argv=None):
         'match_filter': match_filter,
         'no_color': opts.no_color,
         'ffmpeg_location': opts.ffmpeg_location,
+        'hls_prefer_native': opts.hls_prefer_native,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index eff1122c5..9fb66e2f7 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -34,6 +34,9 @@ def get_suitable_downloader(info_dict, params={}):
         if ed.supports(info_dict):
             return ed
 
+    if protocol == 'm3u8' and params.get('hls_prefer_native'):
+        return NativeHlsFD
+
     return PROTOCOL_MAP.get(protocol, HttpFD)
 
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ba35399cf..5f678f76b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -424,6 +424,10 @@ def parseOpts(overrideArguments=None):
         '--xattr-set-filesize',
         dest='xattr_set_filesize', action='store_true',
         help='(experimental) set file xattribute ytdl.filesize with expected filesize')
+    downloader.add_option(
+        '--hls-prefer-native',
+        dest='hls_prefer_native', action='store_true',
+        help='(experimental) Use the native HLS downloader instead of ffmpeg.')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',

From 163d966707a7e49bcdad4ebd9189922b58223395 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 16:21:02 +0100
Subject: [PATCH 901/974] [downloader/external] curl: Add the '--location' flag

curl doesn't follow redirections by default
---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index ff031d2e0..51c41c704 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -75,7 +75,7 @@ class ExternalFD(FileDownloader):
 
 class CurlFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
-        cmd = [self.exe, '-o', tmpfilename]
+        cmd = [self.exe, '--location', '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--interface')

From 662435f7281da418612131512725fa5e367605be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 16:29:24 +0100
Subject: [PATCH 902/974] [YoutubeDL] Use a Request object for getting the
 cookies (fixes #4970)

So that we don't have to implement all the methods used by the cookiejar.
---
 youtube_dl/YoutubeDL.py | 25 ++-----------------------
 1 file changed, 2 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ea2435e0a..0a41152d6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -954,30 +954,9 @@ class YoutubeDL(object):
         return res
 
     def _calc_cookies(self, info_dict):
-        class _PseudoRequest(object):
-            def __init__(self, url):
-                self.url = url
-                self.headers = {}
-                self.unverifiable = False
-
-            def add_unredirected_header(self, k, v):
-                self.headers[k] = v
-
-            def get_full_url(self):
-                return self.url
-
-            def is_unverifiable(self):
-                return self.unverifiable
-
-            def has_header(self, h):
-                return h in self.headers
-
-            def get_header(self, h, default=None):
-                return self.headers.get(h, default)
-
-        pr = _PseudoRequest(info_dict['url'])
+        pr = compat_urllib_request.Request(info_dict['url'])
         self.cookiejar.add_cookie_header(pr)
-        return pr.headers.get('Cookie')
+        return pr.get_header('Cookie')
 
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'

From fbc503d69698fca33525707b859b41d491d681ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 16:40:42 +0100
Subject: [PATCH 903/974] [downloader/hls] Fix detection of ffmpeg/avconv
 (reported in #4966)

---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 8be4f4249..3650d715e 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -23,7 +23,7 @@ class HlsFD(FileDownloader):
         tmpfilename = self.temp_name(filename)
 
         ffpp = FFmpegPostProcessor(downloader=self)
-        if not ffpp.available:
+        if not ffpp.available():
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()

From f740fae2a418ce6c1d229890b48730d257a80dee Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:26:41 +0100
Subject: [PATCH 904/974] [ffmpeg] Make available a property

---
 youtube_dl/YoutubeDL.py            | 2 +-
 youtube_dl/postprocessor/ffmpeg.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ea2435e0a..a74e73b0c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1301,7 +1301,7 @@ class YoutubeDL(object):
                     downloaded = []
                     success = True
                     merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
-                    if not merger.available():
+                    if not merger.available:
                         postprocessors = []
                         self.report_warning('You have requested multiple '
                                             'formats but ffmpeg or avconv are not installed.'
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 16babf6a5..e98fe9834 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -34,7 +34,7 @@ class FFmpegPostProcessor(PostProcessor):
         self._determine_executables()
 
     def check_version(self):
-        if not self.available():
+        if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
         required_version = '10-0' if self._uses_avconv() else '1.0'
@@ -108,6 +108,7 @@ class FFmpegPostProcessor(PostProcessor):
                 self.probe_basename = p
                 break
 
+    @property
     def available(self):
         return self.basename is not None
 

From 65bf37ef83b67bd0e2c594283852d120734b2a3a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:27:29 +0100
Subject: [PATCH 905/974] [ffmpeg] Remove trivial helper method

---
 youtube_dl/postprocessor/ffmpeg.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index e98fe9834..3f2e6cf1d 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -37,7 +37,7 @@ class FFmpegPostProcessor(PostProcessor):
         if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
-        required_version = '10-0' if self._uses_avconv() else '1.0'
+        required_version = '10-0' if self.basename == 'avconv' else '1.0'
         if is_outdated_version(
                 self._versions[self.basename], required_version):
             warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
@@ -112,9 +112,6 @@ class FFmpegPostProcessor(PostProcessor):
     def available(self):
         return self.basename is not None
 
-    def _uses_avconv(self):
-        return self.basename == 'avconv'
-
     @property
     def executable(self):
         return self._paths[self.basename]

From 46312e0b46147ff26008eee62b2f9ba47b30fe73 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:29:32 +0100
Subject: [PATCH 906/974] release 2015.02.17

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d68896202..acd0ef41b 100644
--- a/README.md
+++ b/README.md
@@ -161,6 +161,8 @@ which means you can modify it, redistribute it or use it however you like.
     --playlist-reverse               Download playlist videos in reverse order
     --xattr-set-filesize             (experimental) set file xattribute
                                      ytdl.filesize with expected filesize
+    --hls-prefer-native              (experimental) Use the native HLS
+                                     downloader instead of ffmpeg.
     --external-downloader COMMAND    (experimental) Use the specified external
                                      downloader. Currently supports
                                      aria2c,curl,wget
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index bbe0e53b5..88834e058 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.16.1'
+__version__ = '2015.02.17'

From 8ac27a68e618ecc789bbef4dd0eb408324498fdd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:35:03 +0100
Subject: [PATCH 907/974] [hls] Switch to available as a property

---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 3650d715e..8be4f4249 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -23,7 +23,7 @@ class HlsFD(FileDownloader):
         tmpfilename = self.temp_name(filename)
 
         ffpp = FFmpegPostProcessor(downloader=self)
-        if not ffpp.available():
+        if not ffpp.available:
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()

From 7010577720bbea309566c38adbd912c15400b9fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:35:08 +0100
Subject: [PATCH 908/974] release 2015.02.17.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 88834e058..4ffbd3ae4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.17'
+__version__ = '2015.02.17.1'

From 4069766c527d10b8e25b9262a3882101367deb3e Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Tue, 17 Feb 2015 22:31:35 +0600
Subject: [PATCH 909/974] [extractor/common] Test URLs with GET

---
 youtube_dl/extractor/common.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e74b7bf25..4b0ee882c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -753,9 +753,7 @@ class InfoExtractor(object):
 
     def _is_valid_url(self, url, video_id, item='video'):
         try:
-            self._request_webpage(
-                HEADRequest(url), video_id,
-                'Checking %s URL' % item)
+            self._request_webpage(url, video_id, 'Checking %s URL' % item)
             return True
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):

From 0d93378887bd527b1df04e6138b4bc41382dd48f Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Tue, 17 Feb 2015 22:34:29 +0600
Subject: [PATCH 910/974] [videolecturesnet] Check http format URLs (Closes
 #4968)

---
 youtube_dl/extractor/videolecturesnet.py | 33 +++++++++++++++++-------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index ebd2a3dca..1ec5298e9 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -49,15 +49,30 @@ class VideoLecturesNetIE(InfoExtractor):
         thumbnail = (
             None if thumbnail_el is None else thumbnail_el.attrib.get('src'))
 
-        formats = [{
-            'url': v.attrib['src'],
-            'width': int_or_none(v.attrib.get('width')),
-            'height': int_or_none(v.attrib.get('height')),
-            'filesize': int_or_none(v.attrib.get('size')),
-            'tbr': int_or_none(v.attrib.get('systemBitrate')) / 1000.0,
-            'ext': v.attrib.get('ext'),
-        } for v in switch.findall('./video')
-            if v.attrib.get('proto') == 'http']
+        formats = []
+        for v in switch.findall('./video'):
+            proto = v.attrib.get('proto')
+            if not proto in ['http', 'rtmp']:
+                continue
+            f = {
+                'width': int_or_none(v.attrib.get('width')),
+                'height': int_or_none(v.attrib.get('height')),
+                'filesize': int_or_none(v.attrib.get('size')),
+                'tbr': int_or_none(v.attrib.get('systemBitrate')) / 1000.0,
+                'ext': v.attrib.get('ext'),
+            }
+            src = v.attrib['src']
+            if proto == 'http':
+                if self._is_valid_url(src, video_id):
+                    f['url'] = src
+                    formats.append(f)
+            elif proto == 'rtmp':
+                f.update({
+                    'url': v.attrib['streamer'],
+                    'play_path': src,
+                })
+                formats.append(f)
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From be24c8697f51fbc68e2cd99668e675a07a3735de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:38:31 +0100
Subject: [PATCH 911/974] release 2015.02.17.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4ffbd3ae4..34a8d1184 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.17.1'
+__version__ = '2015.02.17.2'

From 4cd95bcbc34d74e506e6158251d50d83697da5fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 18:55:53 +0100
Subject: [PATCH 912/974] [twitch:stream] Prefer the 'source' format (fixes
 #4972)

---
 youtube_dl/extractor/common.py | 11 +++++++++++
 youtube_dl/extractor/twitch.py |  6 ++++++
 2 files changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4b0ee882c..31599bf52 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -839,6 +839,7 @@ class InfoExtractor(object):
             note='Downloading m3u8 information',
             errnote='Failed to download m3u8 information')
         last_info = None
+        last_media= None
         kv_rex = re.compile(
             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
         for line in m3u8_doc.splitlines():
@@ -849,6 +850,13 @@ class InfoExtractor(object):
                     if v.startswith('"'):
                         v = v[1:-1]
                     last_info[m.group('key')] = v
+            elif line.startswith('#EXT-X-MEDIA:'):
+                last_media = {}
+                for m in kv_rex.finditer(line):
+                    v = m.group('val')
+                    if v.startswith('"'):
+                        v = v[1:-1]
+                    last_media[m.group('key')] = v
             elif line.startswith('#') or not line.strip():
                 continue
             else:
@@ -877,6 +885,9 @@ class InfoExtractor(object):
                     width_str, height_str = resolution.split('x')
                     f['width'] = int(width_str)
                     f['height'] = int(height_str)
+                if last_media is not None:
+                    f['m3u8_media'] = last_media
+                    last_media = None
                 formats.append(f)
                 last_info = {}
         self._sort_formats(formats)
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 87290d002..8e296698e 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -348,6 +348,12 @@ class TwitchStreamIE(TwitchBaseIE):
             '%s/api/channel/hls/%s.m3u8?%s'
             % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
             channel_id, 'mp4')
+        # prefer the 'source' stream, the others are limited to 30 fps
+        def _sort_source(f):
+            if f.get('m3u8_media') is not None and f['m3u8_media'].get('NAME') == 'Source':
+                return 1
+            return 0
+        formats = sorted(formats, key=_sort_source)
 
         view_count = stream.get('viewers')
         timestamp = parse_iso8601(stream.get('created_at'))

From 4a8d4a53b107dde6e78f37fd63dee22b3d29a877 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Feb 2015 01:16:49 +0600
Subject: [PATCH 913/974] [videolecturesnet] Fix rtmp stream glitches (Closes
 #4968)

---
 youtube_dl/extractor/videolecturesnet.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 1ec5298e9..d46234856 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -70,6 +70,7 @@ class VideoLecturesNetIE(InfoExtractor):
                 f.update({
                     'url': v.attrib['streamer'],
                     'play_path': src,
+                    'rtmp_real_time': True,
                 })
                 formats.append(f)
         self._sort_formats(formats)

From 98f000409f072e544d6c3b07809022f703b8b23d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 19:54:04 +0100
Subject: [PATCH 914/974] [radio.de] Fix extraction

---
 youtube_dl/extractor/radiode.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/radiode.py b/youtube_dl/extractor/radiode.py
index f95bc9454..aa5f6f8ad 100644
--- a/youtube_dl/extractor/radiode.py
+++ b/youtube_dl/extractor/radiode.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
 
 
@@ -10,13 +8,13 @@ class RadioDeIE(InfoExtractor):
     _VALID_URL = r'https?://(?P<id>.+?)\.(?:radio\.(?:de|at|fr|pt|es|pl|it)|rad\.io)'
     _TEST = {
         'url': 'http://ndr2.radio.de/',
-        'md5': '3b4cdd011bc59174596b6145cda474a4',
         'info_dict': {
             'id': 'ndr2',
             'ext': 'mp3',
             'title': 're:^NDR 2 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': 'md5:591c49c702db1a33751625ebfb67f273',
             'thumbnail': 're:^https?://.*\.png',
+            'is_live': True,
         },
         'params': {
             'skip_download': True,
@@ -25,16 +23,15 @@ class RadioDeIE(InfoExtractor):
 
     def _real_extract(self, url):
         radio_id = self._match_id(url)
-
         webpage = self._download_webpage(url, radio_id)
+        jscode = self._search_regex(
+            r"'components/station/stationService':\s*\{\s*'?station'?:\s*(\{.*?\s*\}),\n",
+            webpage, 'broadcast')
 
-        broadcast = json.loads(self._search_regex(
-            r'_getBroadcast\s*=\s*function\(\s*\)\s*{\s*return\s+({.+?})\s*;\s*}',
-            webpage, 'broadcast'))
-
+        broadcast = self._parse_json(jscode, radio_id)
         title = self._live_title(broadcast['name'])
         description = broadcast.get('description') or broadcast.get('shortDescription')
-        thumbnail = broadcast.get('picture4Url') or broadcast.get('picture4TransUrl')
+        thumbnail = broadcast.get('picture4Url') or broadcast.get('picture4TransUrl') or broadcast.get('logo100x100')
 
         formats = [{
             'url': stream['streamUrl'],

From 5cda4eda7253d766611363a880af46895c11ad17 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:37:48 +0100
Subject: [PATCH 915/974] [YoutubeDL] Use a progress hook for progress
 reporting

Instead of every downloader calling two helper functions, let our progress report be an ordinary progress hook like everyone else's.
Closes #4875.
---
 youtube_dl/YoutubeDL.py         | 13 +++--
 youtube_dl/downloader/common.py | 87 +++++++++++++++++++++------------
 youtube_dl/downloader/f4m.py    | 70 ++++++++++++++++----------
 youtube_dl/downloader/http.py   | 13 +++--
 youtube_dl/downloader/rtmp.py   | 18 +++----
 5 files changed, 128 insertions(+), 73 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3658332ec..b087d356f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -199,18 +199,25 @@ class YoutubeDL(object):
                        postprocessor.
     progress_hooks:    A list of functions that get called on download
                        progress, with a dictionary with the entries
-                       * status: One of "downloading" and "finished".
+                       * status: One of "downloading", "error", or "finished".
                                  Check this first and ignore unknown values.
 
-                       If status is one of "downloading" or "finished", the
+                       If status is one of "downloading", or "finished", the
                        following properties may also be present:
                        * filename: The final filename (always present)
+                       * tmpfilename: The filename we're currently writing to
                        * downloaded_bytes: Bytes on disk
                        * total_bytes: Size of the whole file, None if unknown
-                       * tmpfilename: The filename we're currently writing to
+                       * total_bytes_estimate: Guess of the eventual file size,
+                                               None if unavailable.
+                       * elapsed: The number of seconds since download started.
                        * eta: The estimated time in seconds, None if unknown
                        * speed: The download speed in bytes/second, None if
                                 unknown
+                       * fragment_index: The counter of the currently
+                                         downloaded video fragment.
+                       * fragment_count: The number of fragments (= individual
+                                         files that will be merged)
 
                        Progress hooks are guaranteed to be called at least once
                        (with status "finished") if the download is successful.
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 7bb3a948d..45e55b99c 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from __future__ import division, unicode_literals
 
 import os
 import re
@@ -54,6 +54,7 @@ class FileDownloader(object):
         self.ydl = ydl
         self._progress_hooks = []
         self.params = params
+        self.add_progress_hook(self.report_progress)
 
     @staticmethod
     def format_seconds(seconds):
@@ -226,42 +227,64 @@ class FileDownloader(object):
             self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
         self.to_console_title('youtube-dl ' + msg)
 
-    def report_progress(self, percent, data_len_str, speed, eta):
-        """Report download progress."""
-        if self.params.get('noprogress', False):
+    def report_progress(self, s):
+        if s['status'] == 'finished':
+            if self.params.get('noprogress', False):
+                self.to_screen('[download] Download completed')
+            else:
+                s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+                if s.get('elapsed') is not None:
+                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
+                    msg_template = '100%% of %(_total_bytes_str)s in %(_elapsed_str)s'
+                else:
+                    msg_template = '100%% of %(_total_bytes_str)s'
+                self._report_progress_status(
+                    msg_template % s, is_last_line=True)
+
+        if self.params.get('noprogress'):
             return
-        if eta is not None:
-            eta_str = self.format_eta(eta)
-        else:
-            eta_str = 'Unknown ETA'
-        if percent is not None:
-            percent_str = self.format_percent(percent)
-        else:
-            percent_str = 'Unknown %'
-        speed_str = self.format_speed(speed)
 
-        msg = ('%s of %s at %s ETA %s' %
-               (percent_str, data_len_str, speed_str, eta_str))
-        self._report_progress_status(msg)
-
-    def report_progress_live_stream(self, downloaded_data_len, speed, elapsed):
-        if self.params.get('noprogress', False):
+        if s['status'] != 'downloading':
             return
-        downloaded_str = format_bytes(downloaded_data_len)
-        speed_str = self.format_speed(speed)
-        elapsed_str = FileDownloader.format_seconds(elapsed)
-        msg = '%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str)
-        self._report_progress_status(msg)
 
-    def report_finish(self, data_len_str, tot_time):
-        """Report download finished."""
-        if self.params.get('noprogress', False):
-            self.to_screen('[download] Download completed')
+        if s.get('eta') is not None:
+            s['_eta_str'] = self.format_eta(s['eta'])
         else:
-            self._report_progress_status(
-                ('100%% of %s in %s' %
-                 (data_len_str, self.format_seconds(tot_time))),
-                is_last_line=True)
+            s['_eta_str'] = 'Unknown ETA'
+
+        if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
+            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
+        elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
+            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
+        else:
+            if s.get('downloaded_bytes') == 0:
+                s['_percent_str'] = self.format_percent(0)
+            else:
+                s['_percent_str'] = 'Unknown %'
+
+        if s.get('speed') is not None:
+            s['_speed_str'] = self.format_speed(s['speed'])
+        else:
+            s['_speed_str'] = 'Unknown speed'
+
+        if s.get('total_bytes') is not None:
+            s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+            msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
+        elif s.get('total_bytes_estimate') is not None:
+            s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
+            msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
+        else:
+            if s.get('downloaded_bytes') is not None:
+                s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
+                if s.get('elapsed'):
+                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
+                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
+                else:
+                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
+            else:
+                msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
+
+        self._report_progress_status(msg_template % s)
 
     def report_resuming_byte(self, resume_len):
         """Report attempt to resume at given byte."""
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 0e7a1c200..5a1f8e680 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from __future__ import division, unicode_literals
 
 import base64
 import io
@@ -252,17 +252,6 @@ class F4mFD(FileDownloader):
         requested_bitrate = info_dict.get('tbr')
         self.to_screen('[download] Downloading f4m manifest')
         manifest = self.ydl.urlopen(man_url).read()
-        self.report_destination(filename)
-        http_dl = HttpQuietDownloader(
-            self.ydl,
-            {
-                'continuedl': True,
-                'quiet': True,
-                'noprogress': True,
-                'ratelimit': self.params.get('ratelimit', None),
-                'test': self.params.get('test', False),
-            }
-        )
 
         doc = etree.fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)
@@ -298,39 +287,67 @@ class F4mFD(FileDownloader):
         # For some akamai manifests we'll need to add a query to the fragment url
         akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))
 
+        self.report_destination(filename)
+        http_dl = HttpQuietDownloader(
+            self.ydl,
+            {
+                'continuedl': True,
+                'quiet': True,
+                'noprogress': True,
+                'ratelimit': self.params.get('ratelimit', None),
+                'test': self.params.get('test', False),
+            }
+        )
         tmpfilename = self.temp_name(filename)
         (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
+
         write_flv_header(dest_stream)
         write_metadata_tag(dest_stream, metadata)
 
         # This dict stores the download progress, it's updated by the progress
         # hook
         state = {
+            'status': 'downloading',
             'downloaded_bytes': 0,
-            'frag_counter': 0,
+            'frag_index': 0,
+            'frag_count': total_frags,
+            'filename': filename,
+            'tmpfilename': tmpfilename,
         }
         start = time.time()
 
-        def frag_progress_hook(status):
-            frag_total_bytes = status.get('total_bytes', 0)
-            estimated_size = (state['downloaded_bytes'] +
-                              (total_frags - state['frag_counter']) * frag_total_bytes)
-            if status['status'] == 'finished':
+        def frag_progress_hook(s):
+            if s['status'] not in ('downloading', 'finished'):
+                return
+
+            frag_total_bytes = s.get('total_bytes', 0)
+            if s['status'] == 'finished':
                 state['downloaded_bytes'] += frag_total_bytes
-                state['frag_counter'] += 1
-                progress = self.calc_percent(state['frag_counter'], total_frags)
+                state['frag_index'] += 1
+
+            estimated_size = (
+                (state['downloaded_bytes'] + frag_total_bytes)
+                / (state['frag_index'] + 1) * total_frags)
+            time_now = time.time()
+            state['total_bytes_estimate'] = estimated_size
+            state['elapsed'] = time_now - start
+
+            if s['status'] == 'finished':
+                progress = self.calc_percent(state['frag_index'], total_frags)
                 byte_counter = state['downloaded_bytes']
             else:
-                frag_downloaded_bytes = status['downloaded_bytes']
+                frag_downloaded_bytes = s['downloaded_bytes']
                 byte_counter = state['downloaded_bytes'] + frag_downloaded_bytes
                 frag_progress = self.calc_percent(frag_downloaded_bytes,
                                                   frag_total_bytes)
-                progress = self.calc_percent(state['frag_counter'], total_frags)
+                progress = self.calc_percent(state['frag_index'], total_frags)
                 progress += frag_progress / float(total_frags)
 
-            eta = self.calc_eta(start, time.time(), estimated_size, byte_counter)
-            self.report_progress(progress, format_bytes(estimated_size),
-                                 status.get('speed'), eta)
+                state['eta'] = self.calc_eta(
+                    start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes)
+                state['speed'] = s.get('speed')
+            self._hook_progress(state)
+
         http_dl.add_progress_hook(frag_progress_hook)
 
         frags_filenames = []
@@ -354,8 +371,8 @@ class F4mFD(FileDownloader):
             frags_filenames.append(frag_filename)
 
         dest_stream.close()
-        self.report_finish(format_bytes(state['downloaded_bytes']), time.time() - start)
 
+        elapsed = time.time() - start
         self.try_rename(tmpfilename, filename)
         for frag_file in frags_filenames:
             os.remove(frag_file)
@@ -366,6 +383,7 @@ class F4mFD(FileDownloader):
             'total_bytes': fsize,
             'filename': filename,
             'status': 'finished',
+            'elapsed': elapsed,
         })
 
         return True
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 49170cf9d..d37522aea 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -200,16 +200,16 @@ class HttpFD(FileDownloader):
             else:
                 percent = self.calc_percent(byte_counter, data_len)
                 eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
-            self.report_progress(percent, data_len_str, speed, eta)
 
             self._hook_progress({
+                'status': 'downloading',
                 'downloaded_bytes': byte_counter,
                 'total_bytes': data_len,
                 'tmpfilename': tmpfilename,
                 'filename': filename,
-                'status': 'downloading',
                 'eta': eta,
                 'speed': speed,
+                'elapsed': now - start,
             })
 
             if is_test and byte_counter == data_len:
@@ -221,7 +221,13 @@ class HttpFD(FileDownloader):
             return False
         if tmpfilename != '-':
             stream.close()
-        self.report_finish(data_len_str, (time.time() - start))
+
+        self._hook_progress({
+            'downloaded_bytes': byte_counter,
+            'total_bytes': data_len,
+            'tmpfilename': tmpfilename,
+            'status': 'error',
+        })
         if data_len is not None and byte_counter != data_len:
             raise ContentTooShortError(byte_counter, int(data_len))
         self.try_rename(tmpfilename, filename)
@@ -235,6 +241,7 @@ class HttpFD(FileDownloader):
             'total_bytes': byte_counter,
             'filename': filename,
             'status': 'finished',
+            'elapsed': time.time() - start,
         })
 
         return True
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index f7eeb6f43..79836fe99 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -51,23 +51,23 @@ class RtmpFD(FileDownloader):
                     if not resume_percent:
                         resume_percent = percent
                         resume_downloaded_data_len = downloaded_data_len
-                    eta = self.calc_eta(start, time.time(), 100 - resume_percent, percent - resume_percent)
-                    speed = self.calc_speed(start, time.time(), downloaded_data_len - resume_downloaded_data_len)
+                    time_now = time.time()
+                    eta = self.calc_eta(start, time_now, 100 - resume_percent, percent - resume_percent)
+                    speed = self.calc_speed(start, time_now, downloaded_data_len - resume_downloaded_data_len)
                     data_len = None
                     if percent > 0:
                         data_len = int(downloaded_data_len * 100 / percent)
-                    data_len_str = '~' + format_bytes(data_len)
-                    self.report_progress(percent, data_len_str, speed, eta)
-                    cursor_in_new_line = False
                     self._hook_progress({
+                        'status': 'downloading',
                         'downloaded_bytes': downloaded_data_len,
-                        'total_bytes': data_len,
+                        'total_bytes_estimate': data_len,
                         'tmpfilename': tmpfilename,
                         'filename': filename,
-                        'status': 'downloading',
                         'eta': eta,
+                        'elapsed': time_now - start,
                         'speed': speed,
                     })
+                    cursor_in_new_line = False
                 else:
                     # no percent for live streams
                     mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line)
@@ -75,15 +75,15 @@ class RtmpFD(FileDownloader):
                         downloaded_data_len = int(float(mobj.group(1)) * 1024)
                         time_now = time.time()
                         speed = self.calc_speed(start, time_now, downloaded_data_len)
-                        self.report_progress_live_stream(downloaded_data_len, speed, time_now - start)
-                        cursor_in_new_line = False
                         self._hook_progress({
                             'downloaded_bytes': downloaded_data_len,
                             'tmpfilename': tmpfilename,
                             'filename': filename,
                             'status': 'downloading',
+                            'elapsed': time_now - start,
                             'speed': speed,
                         })
+                        cursor_in_new_line = False
                     elif self.params.get('verbose', False):
                         if not cursor_in_new_line:
                             self.to_screen('')

From 16e7711e22648027739096560914a976b8eea786 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:42:31 +0100
Subject: [PATCH 916/974] [downloader/http] Remove gruesome import

---
 youtube_dl/downloader/http.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 49170cf9d..25032ad4e 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
-import os
-import time
-
-from socket import error as SocketError
 import errno
+import os
+import socket
+import time
 
 from .common import FileDownloader
 from ..compat import (
@@ -102,7 +101,7 @@ class HttpFD(FileDownloader):
                             resume_len = 0
                             open_mode = 'wb'
                             break
-            except SocketError as e:
+            except socket.error as e:
                 if e.errno != errno.ECONNRESET:
                     # Connection reset is no problem, just retry
                     raise

From a91a2c1a83fdd195e850d4ad9c298c01a145ebf0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:44:41 +0100
Subject: [PATCH 917/974] [downloader] Remove various unneeded assignments and
 imports

---
 youtube_dl/downloader/f4m.py  | 3 ---
 youtube_dl/downloader/http.py | 5 +----
 youtube_dl/downloader/rtmp.py | 1 -
 3 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 5a1f8e680..b40ebfa50 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -15,7 +15,6 @@ from ..compat import (
 from ..utils import (
     struct_pack,
     struct_unpack,
-    format_bytes,
     encodeFilename,
     sanitize_open,
     xpath_text,
@@ -334,10 +333,8 @@ class F4mFD(FileDownloader):
 
             if s['status'] == 'finished':
                 progress = self.calc_percent(state['frag_index'], total_frags)
-                byte_counter = state['downloaded_bytes']
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
-                byte_counter = state['downloaded_bytes'] + frag_downloaded_bytes
                 frag_progress = self.calc_percent(frag_downloaded_bytes,
                                                   frag_total_bytes)
                 progress = self.calc_percent(state['frag_index'], total_frags)
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index d37522aea..6dec528f2 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -15,7 +15,6 @@ from ..utils import (
     ContentTooShortError,
     encodeFilename,
     sanitize_open,
-    format_bytes,
 )
 
 
@@ -137,7 +136,6 @@ class HttpFD(FileDownloader):
                 self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
                 return False
 
-        data_len_str = format_bytes(data_len)
         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)
         start = time.time()
@@ -196,9 +194,8 @@ class HttpFD(FileDownloader):
             # Progress message
             speed = self.calc_speed(start, now, byte_counter - resume_len)
             if data_len is None:
-                eta = percent = None
+                eta = None
             else:
-                percent = self.calc_percent(byte_counter, data_len)
                 eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 
             self._hook_progress({
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 79836fe99..0a52c34c7 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -11,7 +11,6 @@ from ..compat import compat_str
 from ..utils import (
     check_executable,
     encodeFilename,
-    format_bytes,
     get_exe_version,
 )
 

From fa156077737d5162795a221fe2995a276d31c6d3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:46:20 +0100
Subject: [PATCH 918/974] PEP8 fixes

---
 youtube_dl/extractor/common.py           | 3 +--
 youtube_dl/extractor/twitch.py           | 1 +
 youtube_dl/extractor/videolecturesnet.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 31599bf52..08b8ad37c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -27,7 +27,6 @@ from ..utils import (
     compiled_regex_type,
     ExtractorError,
     float_or_none,
-    HEADRequest,
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,
@@ -839,7 +838,7 @@ class InfoExtractor(object):
             note='Downloading m3u8 information',
             errnote='Failed to download m3u8 information')
         last_info = None
-        last_media= None
+        last_media = None
         kv_rex = re.compile(
             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
         for line in m3u8_doc.splitlines():
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8e296698e..4b0d8988d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -348,6 +348,7 @@ class TwitchStreamIE(TwitchBaseIE):
             '%s/api/channel/hls/%s.m3u8?%s'
             % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
             channel_id, 'mp4')
+
         # prefer the 'source' stream, the others are limited to 30 fps
         def _sort_source(f):
             if f.get('m3u8_media') is not None and f['m3u8_media'].get('NAME') == 'Source':
diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index d46234856..d6a7eb203 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -52,7 +52,7 @@ class VideoLecturesNetIE(InfoExtractor):
         formats = []
         for v in switch.findall('./video'):
             proto = v.attrib.get('proto')
-            if not proto in ['http', 'rtmp']:
+            if proto not in ['http', 'rtmp']:
                 continue
             f = {
                 'width': int_or_none(v.attrib.get('width')),

From 30965ac66a4e33b8e70d6c0cf5da1a746054d6cd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:27:57 +0100
Subject: [PATCH 919/974] [vimeo] Prevent infinite loops if video password
 verification fails
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We're seeing this in the tests¹ right now, which do not terminate.

¹  https://travis-ci.org/jaimeMF/youtube-dl/jobs/51135858
---
 youtube_dl/extractor/vimeo.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 303e81447..1959d5e36 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -18,6 +18,7 @@ from ..utils import (
     InAdvancePagedList,
     int_or_none,
     RegexNotFoundError,
+    smuggle_url,
     std_headers,
     unsmuggle_url,
     urlencode_postdata,
@@ -267,8 +268,11 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
                 raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
 
             if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
+                if data and '_video_password_verified' in data:
+                    raise ExtractorError('video password verification failed!')
                 self._verify_video_password(url, video_id, webpage)
-                return self._real_extract(url)
+                return self._real_extract(
+                    smuggle_url(url, {'_video_password_verified': 'verified'}))
             else:
                 raise ExtractorError('Unable to extract info section',
                                      cause=e)

From a3fa5da4967bf2699bfd982cf56a8f627a13bda4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:33:31 +0100
Subject: [PATCH 920/974] [vimeo] Amend playlist tests

---
 youtube_dl/extractor/vimeo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 1959d5e36..e7284049d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -405,6 +405,7 @@ class VimeoChannelIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://vimeo.com/channels/tributes',
         'info_dict': {
+            'id': 'tributes',
             'title': 'Vimeo Tributes',
         },
         'playlist_mincount': 25,
@@ -483,6 +484,7 @@ class VimeoUserIE(VimeoChannelIE):
         'url': 'http://vimeo.com/nkistudio/videos',
         'info_dict': {
             'title': 'Nki',
+            'id': 'nkistudio',
         },
         'playlist_mincount': 66,
     }]
@@ -500,6 +502,7 @@ class VimeoAlbumIE(VimeoChannelIE):
     _TESTS = [{
         'url': 'http://vimeo.com/album/2632481',
         'info_dict': {
+            'id': '2632481',
             'title': 'Staff Favorites: November 2013',
         },
         'playlist_mincount': 13,
@@ -530,6 +533,7 @@ class VimeoGroupsIE(VimeoAlbumIE):
     _TESTS = [{
         'url': 'http://vimeo.com/groups/rolexawards',
         'info_dict': {
+            'id': 'rolexawards',
             'title': 'Rolex Awards for Enterprise',
         },
         'playlist_mincount': 73,
@@ -612,6 +616,7 @@ class VimeoLikesIE(InfoExtractor):
         'url': 'https://vimeo.com/user755559/likes/',
         'playlist_mincount': 293,
         "info_dict": {
+            'id': 'user755559_likes',
             "description": "See all the videos urza likes",
             "title": 'Videos urza likes',
         },

From 15ec669374727c9bdc3aad0df141fdf305cb80be Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:33:41 +0100
Subject: [PATCH 921/974] [vk] Amend playlist test

---
 youtube_dl/extractor/vk.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 81e02a624..7dea8c59d 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -217,6 +217,9 @@ class VKUserVideosIE(InfoExtractor):
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TEST = {
         'url': 'http://vk.com/videos205387401',
+        'info_dict': {
+            'id': '205387401',
+        },
         'playlist_mincount': 4,
     }
 

From 1def5f359ef6e197e4cc5d610ed86e109e240080 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:34:45 +0100
Subject: [PATCH 922/974] [livestream] Correct playlist ID and add a test for
 it

---
 youtube_dl/extractor/livestream.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 5247c6f58..3642089f7 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -37,6 +37,7 @@ class LivestreamIE(InfoExtractor):
         'url': 'http://new.livestream.com/tedx/cityenglish',
         'info_dict': {
             'title': 'TEDCity2.0 (English)',
+            'id': '2245590',
         },
         'playlist_mincount': 4,
     }, {
@@ -148,7 +149,8 @@ class LivestreamIE(InfoExtractor):
                   if is_relevant(video_data, video_id)]
         if video_id is None:
             # This is an event page:
-            return self.playlist_result(videos, info['id'], info['full_name'])
+            return self.playlist_result(
+                videos, '%s' % info['id'], info['full_name'])
         else:
             if not videos:
                 raise ExtractorError('Cannot find video %s' % video_id)

From 6f53c63df66d95ea684f84ef2f7f0e697acc1d20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:37:48 +0100
Subject: [PATCH 923/974] [test/helper] Only output a newline for forgotten
 keys if keys are really missing

---
 test/helper.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 651ef99b9..fef2db284 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -163,7 +163,9 @@ def expect_info_dict(self, got_dict, expected_dict):
             info_dict_str += ''.join(
                 '    %s: %s,\n' % (_repr(k), _repr(v))
                 for k, v in test_info_dict.items() if k not in missing_keys)
-            info_dict_str += '\n'
+
+            if info_dict_str:
+                info_dict_str += '\n'
         info_dict_str += ''.join(
             '    %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
             for k in missing_keys)

From f21e915fb9ad53a62df65e5c75c984243aaf0aea Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:38:42 +0100
Subject: [PATCH 924/974] [test/helper] Render info_dict with a final comma

---
 test/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index fef2db284..570c91440 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -170,7 +170,7 @@ def expect_info_dict(self, got_dict, expected_dict):
             '    %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
             for k in missing_keys)
         write_string(
-            '\n\'info_dict\': {\n' + info_dict_str + '}\n', out=sys.stderr)
+            '\n\'info_dict\': {\n' + info_dict_str + '},\n', out=sys.stderr)
         self.assertFalse(
             missing_keys,
             'Missing keys in test definition: %s' % (

From 4d278fde645a5ab70b046293a26c12bba146acb0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:38:55 +0100
Subject: [PATCH 925/974] [ign] Amend playlist test

---
 youtube_dl/extractor/ign.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index 3db668cd0..3aade9e74 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -34,6 +34,9 @@ class IGNIE(InfoExtractor):
         },
         {
             'url': 'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind',
+            'info_dict': {
+                'id': '100-little-things-in-gta-5-that-will-blow-your-mind',
+            },
             'playlist': [
                 {
                     'info_dict': {

From 6e99868e4c73468734f3552d430ba6375c4e0880 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:41:45 +0100
Subject: [PATCH 926/974] [buzzfeed] Fix playlist test case

---
 youtube_dl/extractor/buzzfeed.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/buzzfeed.py b/youtube_dl/extractor/buzzfeed.py
index a5d2af174..df503ecc0 100644
--- a/youtube_dl/extractor/buzzfeed.py
+++ b/youtube_dl/extractor/buzzfeed.py
@@ -33,6 +33,7 @@ class BuzzFeedIE(InfoExtractor):
             'skip_download': True,  # Got enough YouTube download tests
         },
         'info_dict': {
+            'id': 'look-at-this-cute-dog-omg',
             'description': 're:Munchkin the Teddy Bear is back ?!',
             'title': 'You Need To Stop What You\'re Doing And Watching This Dog Walk On A Treadmill',
         },
@@ -42,8 +43,8 @@ class BuzzFeedIE(InfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20141124',
                 'uploader_id': 'CindysMunchkin',
-                'description': 're:© 2014 Munchkin the Shih Tzu',
-                'uploader': 'Munchkin the Shih Tzu',
+                'description': 're:© 2014 Munchkin the',
+                'uploader': 're:^Munchkin the',
                 'title': 're:Munchkin the Teddy Bear gets her exercise',
             },
         }]

From 72c1f8de06f11aea670039a17ca63146a1b85af2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:48:52 +0100
Subject: [PATCH 927/974] [bandcamp:album] Fix extractor results and associated
 test

---
 youtube_dl/extractor/bandcamp.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 490cc961a..869294967 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -109,7 +109,7 @@ class BandcampIE(InfoExtractor):
 
 class BandcampAlbumIE(InfoExtractor):
     IE_NAME = 'Bandcamp:album'
-    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+)|/?(?:$|[?#]))'
+    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<album_id>[^?#]+)|/?(?:$|[?#]))'
 
     _TESTS = [{
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
@@ -133,31 +133,37 @@ class BandcampAlbumIE(InfoExtractor):
         ],
         'info_dict': {
             'title': 'Jazz Format Mixtape vol.1',
+            'id': 'jazz-format-mixtape-vol-1',
+            'uploader_id': 'blazo',
         },
         'params': {
             'playlistend': 2
         },
-        'skip': 'Bandcamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test'
+        'skip': 'Bandcamp imposes download limits.'
     }, {
         'url': 'http://nightbringer.bandcamp.com/album/hierophany-of-the-open-grave',
         'info_dict': {
             'title': 'Hierophany of the Open Grave',
+            'uploader_id': 'nightbringer',
+            'id': 'hierophany-of-the-open-grave',
         },
         'playlist_mincount': 9,
     }, {
         'url': 'http://dotscale.bandcamp.com',
         'info_dict': {
             'title': 'Loom',
+            'id': 'dotscale',
+            'uploader_id': 'dotscale',
         },
         'playlist_mincount': 7,
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('subdomain')
-        title = mobj.group('title')
-        display_id = title or playlist_id
-        webpage = self._download_webpage(url, display_id)
+        uploader_id = mobj.group('subdomain')
+        album_id = mobj.group('album_id')
+        playlist_id = album_id or uploader_id
+        webpage = self._download_webpage(url, playlist_id)
         tracks_paths = re.findall(r'<a href="(.*?)" itemprop="url">', webpage)
         if not tracks_paths:
             raise ExtractorError('The page doesn\'t contain any tracks')
@@ -168,8 +174,8 @@ class BandcampAlbumIE(InfoExtractor):
             r'album_title\s*:\s*"(.*?)"', webpage, 'title', fatal=False)
         return {
             '_type': 'playlist',
+            'uploader_id': uploader_id,
             'id': playlist_id,
-            'display_id': display_id,
             'title': title,
             'entries': entries,
         }

From 11e611a7fa809a447939e8a98c8549f2827c862f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:49:10 +0100
Subject: [PATCH 928/974] Extend various playlist tests

---
 youtube_dl/extractor/adultswim.py     | 2 ++
 youtube_dl/extractor/appletrailers.py | 3 +++
 youtube_dl/extractor/dailymotion.py   | 1 +
 youtube_dl/extractor/generic.py       | 1 +
 4 files changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 502a9c25a..34b8b0115 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -38,6 +38,7 @@ class AdultSwimIE(InfoExtractor):
             },
         ],
         'info_dict': {
+            'id': 'rQxZvXQ4ROaSOqq-or2Mow',
             'title': 'Rick and Morty - Pilot',
             'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
         }
@@ -55,6 +56,7 @@ class AdultSwimIE(InfoExtractor):
             }
         ],
         'info_dict': {
+            'id': '-t8CamQlQ2aYZ49ItZCFog',
             'title': 'American Dad - Putting Francine Out of Business',
             'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
         },
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 287f71e07..43e82847f 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -14,6 +14,9 @@ class AppleTrailersIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TEST = {
         "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
+        'info_dict': {
+            'id': 'manofsteel',
+        },
         "playlist": [
             {
                 "md5": "d97a8e575432dbcb81b7c3acb741f8a8",
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index cf5841a7c..b2dbf4a92 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -194,6 +194,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
         'url': 'http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q',
         'info_dict': {
             'title': 'SPORT',
+            'id': 'xv4bw_nqtv_sport',
         },
         'playlist_mincount': 20,
     }]
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c4ba25a96..36a1f65bf 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -473,6 +473,7 @@ class GenericIE(InfoExtractor):
         {
             'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
             'info_dict': {
+                'id': '1986',
                 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
             },
             'playlist_mincount': 2,

From 9cad27008b88b5ba3dfb1d747d7476b05582f385 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:49:34 +0100
Subject: [PATCH 929/974] release 2015.02.18

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 34a8d1184..635b6bdb8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.17.2'
+__version__ = '2015.02.18'

From 6d1a55a521b524dd0c562e3855f81434afb6f807 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:39:14 +0100
Subject: [PATCH 930/974] [youtube] Show entire player URL when -v is given

---
 youtube_dl/extractor/youtube.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 35ef4c303..2c02da2b7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -541,16 +541,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         if cache_spec is not None:
             return lambda s: ''.join(s[i] for i in cache_spec)
 
+        download_note = (
+            'Downloading player %s' % player_url
+            if self._downloader.params.get('verbose') else
+            'Downloading %s player %s' % (player_type, player_id)
+        )
         if player_type == 'js':
             code = self._download_webpage(
                 player_url, video_id,
-                note='Downloading %s player %s' % (player_type, player_id),
+                note=download_note,
                 errnote='Download of %s failed' % player_url)
             res = self._parse_sig_js(code)
         elif player_type == 'swf':
             urlh = self._request_webpage(
                 player_url, video_id,
-                note='Downloading %s player %s' % (player_type, player_id),
+                note=download_note,
                 errnote='Download of %s failed' % player_url)
             code = urlh.read()
             res = self._parse_sig_swf(code)

From 785521bf4fbd99b2916bdab5d847d84424196c1d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:42:23 +0100
Subject: [PATCH 931/974] [youtube] Remove useless if

---
 youtube_dl/extractor/youtube.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2c02da2b7..3d3d43491 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -562,10 +562,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         else:
             assert False, 'Invalid player type %r' % player_type
 
-        if cache_spec is None:
-            test_string = ''.join(map(compat_chr, range(len(example_sig))))
-            cache_res = res(test_string)
-            cache_spec = [ord(c) for c in cache_res]
+        test_string = ''.join(map(compat_chr, range(len(example_sig))))
+        cache_res = res(test_string)
+        cache_spec = [ord(c) for c in cache_res]
 
         self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
         return res

From 3eff81fbf72939abf885afcca2f1155cc976999b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:47:40 +0100
Subject: [PATCH 932/974] [jsinterp] Disable comment support

We need a proper lexer to be able to understand YouTube's code, which contains /* inside of strings.
For now it's sufficient to just disable comment support altogether.

Fixes #4976, fixes #4979, fixes #4980, fixes #4981, fixes #4982.
Closes #4977.
---
 test/test_jsinterp.py          | 11 +++++++++++
 test/test_youtube_signature.py |  6 ++++++
 youtube_dl/jsinterp.py         |  5 +----
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index b91b8c492..fc73e5dc2 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -70,6 +70,8 @@ class TestJSInterpreter(unittest.TestCase):
         self.assertEqual(jsi.call_function('f'), -11)
 
     def test_comments(self):
+        'Skipping: Not yet fully implemented'
+        return
         jsi = JSInterpreter('''
         function x() {
             var x = /* 1 + */ 2;
@@ -80,6 +82,15 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x'), 52)
 
+        jsi = JSInterpreter('''
+        function f() {
+            var x = "/*";
+            var y = 1 /* comment */ + 2;
+            return y;
+        }
+        ''')
+        self.assertEqual(jsi.call_function('f'), 3)
+
     def test_precedence(self):
         jsi = JSInterpreter('''
         function x() {
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 09696e19a..060864434 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -64,6 +64,12 @@ _TESTS = [
         'js',
         '4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288',
         '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B'
+    ),
+    (
+        'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
+        'js',
+        '312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
+        '112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
     )
 ]
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 453e2732c..0e0c7d90d 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -30,13 +30,10 @@ class JSInterpreter(object):
     def __init__(self, code, objects=None):
         if objects is None:
             objects = {}
-        self.code = self._remove_comments(code)
+        self.code = code
         self._functions = {}
         self._objects = objects
 
-    def _remove_comments(self, code):
-        return re.sub(r'(?s)/\*.*?\*/', '', code)
-
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise ExtractorError('Recursion limit reached')

From b8b928d5cb5d21390690cc21c29a3d679605015f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:54:45 +0100
Subject: [PATCH 933/974] [README] Add an FAQ entry for the player change in
 anticipation of many more bug reports

---
 README.md | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index acd0ef41b..a2c148311 100644
--- a/README.md
+++ b/README.md
@@ -515,11 +515,15 @@ If you want to play the video on a machine that is not running youtube-dl, you c
 
 ### ERROR: no fmt_url_map or conn information found in video info
 
-youtube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`.
+YouTube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### ERROR: unable to download video ###
 
-youtube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`.
+YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
+
+### ExtractorError: Could not find JS function u'OF'
+
+In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### SyntaxError: Non-ASCII character ###
 

From 81975f4693c15c1f97ad92e7fafb1463ed2aa4c1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:54:56 +0100
Subject: [PATCH 934/974] release 2015.02.18.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 635b6bdb8..1df05a6df 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.18'
+__version__ = '2015.02.18.1'

From d47c26e16803abc1d15a677d88bbee78f7680db6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 19:56:10 +0100
Subject: [PATCH 935/974] [brightcove] Correct keys in playlists

---
 youtube_dl/extractor/brightcove.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index ea0969d4d..0733bece7 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -95,6 +95,7 @@ class BrightcoveIE(InfoExtractor):
             'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=3550052898001&playerKey=AQ%7E%7E%2CAAABmA9XpXk%7E%2C-Kp7jNgisre1fG5OdqpAFUTcs0lP_ZoL',
             'info_dict': {
                 'title': 'Sealife',
+                'id': '3550319591001',
             },
             'playlist_mincount': 7,
         },
@@ -247,7 +248,7 @@ class BrightcoveIE(InfoExtractor):
         playlist_info = json_data['videoList']
         videos = [self._extract_video_info(video_info) for video_info in playlist_info['mediaCollectionDTO']['videoDTOs']]
 
-        return self.playlist_result(videos, playlist_id=playlist_info['id'],
+        return self.playlist_result(videos, playlist_id='%s' % playlist_info['id'],
                                     playlist_title=playlist_info['mediaCollectionDTO']['displayName'])
 
     def _extract_video_info(self, video_info):

From d8443cd3f7e50fafb918cff5e1676b8d6fef1a64 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 19:56:24 +0100
Subject: [PATCH 936/974] [wsj] Correct test case

---
 youtube_dl/extractor/wsj.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index cbe3dc7be..2ddf29a69 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -18,8 +18,8 @@ class WSJIE(InfoExtractor):
             'id': '1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
             'ext': 'mp4',
             'upload_date': '20150202',
-            'uploader_id': 'bbright',
-            'creator': 'bbright',
+            'uploader_id': 'jdesai',
+            'creator': 'jdesai',
             'categories': list,  # a long list
             'duration': 90,
             'title': 'Bills Coach Rex Ryan Updates His Old Jets Tattoo',

From 9789d7535d6091f64dd688b0bb1f9869b6c6ca49 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 19:58:41 +0100
Subject: [PATCH 937/974] [xtube] Fix test case

---
 test/helper.py                | 10 ++++++++++
 youtube_dl/extractor/xtube.py |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 570c91440..12afdf184 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -113,6 +113,16 @@ def expect_info_dict(self, got_dict, expected_dict):
             self.assertTrue(
                 got.startswith(start_str),
                 'field %s (value: %r) should start with %r' % (info_field, got, start_str))
+        elif isinstance(expected, compat_str) and expected.startswith('contains:'):
+            got = got_dict.get(info_field)
+            contains_str = expected[len('contains:'):]
+            self.assertTrue(
+                isinstance(got, compat_str),
+                'Expected a %s object, but got %s for field %s' % (
+                    compat_str.__name__, type(got).__name__, info_field))
+            self.assertTrue(
+                contains_str in got,
+                'field %s (value: %r) should contain %r' % (info_field, got, contains_str))
         elif isinstance(expected, type):
             got = got_dict.get(info_field)
             self.assertTrue(isinstance(got, expected),
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index e8490b028..1644f53c8 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -22,7 +22,7 @@ class XTubeIE(InfoExtractor):
             'id': 'kVTUy_G222_',
             'ext': 'mp4',
             'title': 'strange erotica',
-            'description': 'http://www.xtube.com an ET kind of thing',
+            'description': 'contains:an ET kind of thing',
             'uploader': 'greenshowers',
             'duration': 450,
             'age_limit': 18,

From 85d586617750d38d742a24f141b099f6b898d269 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 20:03:00 +0100
Subject: [PATCH 938/974] [yahoo] Remove md5sum from test case

The md5 sum has changed repeatedly, and we check whether it looks like a video anyways nowadays.
---
 youtube_dl/extractor/yahoo.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index f8e7041a0..97dbac4cc 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -24,7 +24,6 @@ class YahooIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
-            'md5': '4962b075c08be8690a922ee026d05e69',
             'info_dict': {
                 'id': '2d25e626-2378-391f-ada0-ddaf1417e588',
                 'ext': 'mp4',

From 93a16ba238c25392ed66aa0f1d7cbd3980c45f2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 22:00:12 +0100
Subject: [PATCH 939/974] [vimeo] Raise the ExtractorError with expected=True
 when no video password is given

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index e7284049d..78d287e0e 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -175,7 +175,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
     def _verify_video_password(self, url, video_id, webpage):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
-            raise ExtractorError('This video is protected by a password, use the --video-password option')
+            raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
         token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
         data = compat_urllib_parse.urlencode({
             'password': password,

From d305dd73a3d6927f0a2c63d08662a183fa173833 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 23:59:50 +0100
Subject: [PATCH 940/974] [utils] Fix js_to_json

Previously, the runtime could be atrocious for longer inputs.
---
 test/test_utils.py  | 4 ++++
 youtube_dl/utils.py | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 1c29d0889..c7373af1e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -370,6 +370,10 @@ class TestUtil(unittest.TestCase):
             "playlist":[{"controls":{"all":null}}]
         }''')
 
+        inp = '"SAND Number: SAND 2013-7800P\\nPresenter: Tom Russo\\nHabanero Software Training - Xyce Software\\nXyce, Sandia\\u0027s"'
+        json_code = js_to_json(inp)
+        self.assertEqual(json.loads(json_code), json.loads(inp))
+
     def test_js_to_json_edgecases(self):
         on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}")
         self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3eb6bc6d4..4358137a0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1560,8 +1560,8 @@ def js_to_json(code):
         return '"%s"' % v
 
     res = re.sub(r'''(?x)
-        "(?:[^"\\]*(?:\\\\|\\")?)*"|
-        '(?:[^'\\]*(?:\\\\|\\')?)*'|
+        "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
+        '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
         [a-zA-Z_][.a-zA-Z_0-9]*
         ''', fix_kv, code)
     res = re.sub(r',(\s*\])', lambda m: m.group(1), res)

From b81a359eb67aea83799f1306a441fd6163bf1840 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:28:58 +0100
Subject: [PATCH 941/974] [YoutubeDL] Use render_table for format listing

---
 youtube_dl/YoutubeDL.py | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b087d356f..88809783b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1534,29 +1534,18 @@ class YoutubeDL(object):
         return res
 
     def list_formats(self, info_dict):
-        def line(format, idlen=20):
-            return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
-                format['format_id'],
-                format['ext'],
-                self.format_resolution(format),
-                self._format_note(format),
-            ))
-
         formats = info_dict.get('formats', [info_dict])
-        idlen = max(len('format code'),
-                    max(len(f['format_id']) for f in formats))
-        formats_s = [
-            line(f, idlen) for f in formats
+        table = [
+            [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
+            for f in formats
             if f.get('preference') is None or f['preference'] >= -1000]
         if len(formats) > 1:
-            formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
+            table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
 
-        header_line = line({
-            'format_id': 'format code', 'ext': 'extension',
-            'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
+        header_line = ['format code', 'extension', 'resolution', 'note']
         self.to_screen(
-            '[info] Available formats for %s:\n%s\n%s' %
-            (info_dict['id'], header_line, '\n'.join(formats_s)))
+            '[info] Available formats for %s:\n%s' %
+            (info_dict['id'], render_table(header_line, table)))
 
     def list_thumbnails(self, info_dict):
         thumbnails = info_dict.get('thumbnails')

From c460bdd56b0420bb5175344c0f6523675fdae415 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:31:01 +0100
Subject: [PATCH 942/974] [sandia] Add new extractor (#4974)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/sandia.py   | 117 +++++++++++++++++++++++++++++++
 youtube_dl/utils.py              |  11 ++-
 3 files changed, 128 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/sandia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7d0909b5d..c712b907f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -386,6 +386,7 @@ from .rutube import (
     RutubePersonIE,
 )
 from .rutv import RUTVIE
+from .sandia import SandiaIE
 from .sapo import SapoIE
 from .savefrom import SaveFromIE
 from .sbs import SBSIE
diff --git a/youtube_dl/extractor/sandia.py b/youtube_dl/extractor/sandia.py
new file mode 100644
index 000000000..9c88167f0
--- /dev/null
+++ b/youtube_dl/extractor/sandia.py
@@ -0,0 +1,117 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import itertools
+import json
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+    unified_strdate,
+)
+
+
+class SandiaIE(InfoExtractor):
+    IE_DESC = 'Sandia National Laboratories'
+    _VALID_URL = r'https?://digitalops\.sandia\.gov/Mediasite/Play/(?P<id>[0-9a-f]+)'
+    _TEST = {
+        'url': 'http://digitalops.sandia.gov/Mediasite/Play/24aace4429fc450fb5b38cdbf424a66e1d',
+        'md5': '9422edc9b9a60151727e4b6d8bef393d',
+        'info_dict': {
+            'id': '24aace4429fc450fb5b38cdbf424a66e1d',
+            'ext': 'mp4',
+            'title': 'Xyce Software Training - Section 1',
+            'description': 're:(?s)SAND Number: SAND 2013-7800.{200,}',
+            'upload_date': '20120904',
+            'duration': 7794,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        req = compat_urllib_request.Request(url)
+        req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4')
+        webpage = self._download_webpage(req, video_id)
+
+        js_path = self._search_regex(
+            r'<script type="text/javascript" src="(/Mediasite/FileServer/Presentation/[^"]+)"',
+            webpage, 'JS code URL')
+        js_url = compat_urlparse.urljoin(url, js_path)
+
+        js_code = self._download_webpage(
+            js_url, video_id, note='Downloading player')
+
+        def extract_str(key, **args):
+            return self._search_regex(
+                r'Mediasite\.PlaybackManifest\.%s\s*=\s*(.+);\s*?\n' % re.escape(key),
+                js_code, key, **args)
+
+        def extract_data(key, **args):
+            data_json = extract_str(key, **args)
+            if data_json is None:
+                return data_json
+            return self._parse_json(
+                data_json, video_id, transform_source=js_to_json)
+
+        formats = []
+        for i in itertools.count():
+            fd = extract_data('VideoUrls[%d]' % i, default=None)
+            if fd is None:
+                break
+            formats.append({
+                'format_id': '%s' % i,
+                'format_note': fd['MimeType'].partition('/')[2],
+                'ext': mimetype2ext(fd['MimeType']),
+                'url': fd['Location'],
+                'protocol': 'f4m' if fd['MimeType'] == 'video/x-mp4-fragmented' else None,
+            })
+        self._sort_formats(formats)
+
+        slide_baseurl = compat_urlparse.urljoin(
+            url, extract_data('SlideBaseUrl'))
+        slide_template = slide_baseurl + re.sub(
+            r'\{0:D?([0-9+])\}', r'%0\1d', extract_data('SlideImageFileNameTemplate'))
+        slides = []
+        last_slide_time = 0
+        for i in itertools.count(1):
+            sd = extract_str('Slides[%d]' % i, default=None)
+            if sd is None:
+                break
+            timestamp = int_or_none(self._search_regex(
+                r'^Mediasite\.PlaybackManifest\.CreateSlide\("[^"]*"\s*,\s*([0-9]+),',
+                sd, 'slide %s timestamp' % i, fatal=False))
+            slides.append({
+                'url': slide_template % i,
+                'duration': timestamp - last_slide_time,
+            })
+            last_slide_time = timestamp
+        formats.append({
+            'format_id': 'slides',
+            'protocol': 'slideshow',
+            'url': json.dumps(slides),
+            'preference': -10000,  # Downloader not yet written
+        })
+        self._sort_formats(formats)
+
+        title = extract_data('Title')
+        description = extract_data('Description', fatal=False)
+        duration = int_or_none(extract_data(
+            'Duration', fatal=False), scale=1000)
+        upload_date = unified_strdate(extract_data('AirDate', fatal=False))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+            'upload_date': upload_date,
+            'duration': duration,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4358137a0..238b6556b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1616,6 +1616,15 @@ def args_to_str(args):
     return ' '.join(shlex_quote(a) for a in args)
 
 
+def mimetype2ext(mt):
+    _, _, res = mt.rpartition('/')
+
+    return {
+        'x-ms-wmv': 'wmv',
+        'x-mp4-fragmented': 'mp4',
+    }.get(res, res)
+
+
 def urlhandle_detect_ext(url_handle):
     try:
         url_handle.headers
@@ -1631,7 +1640,7 @@ def urlhandle_detect_ext(url_handle):
             if e:
                 return e
 
-    return getheader('Content-Type').split("/")[1]
+    return mimetype2ext(getheader('Content-Type'))
 
 
 def age_restricted(content_limit, age_limit):

From a025d3c5a529a3343d40886e9783fde0bf997b3f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:31:23 +0100
Subject: [PATCH 943/974] release 2015.02.19

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 4bb68fdc5..e28d7d47c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -352,6 +352,7 @@
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
+ - **Sandia**: Sandia National Laboratories
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1df05a6df..8e6ff49d0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.18.1'
+__version__ = '2015.02.19'

From 770700404365be7b16de89c75825f04e2cf15658 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:38:05 +0100
Subject: [PATCH 944/974] [patreon] Modernize

---
 youtube_dl/extractor/patreon.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index 5429592a7..e5c9822c5 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class PatreonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(.+)'
+    _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(?P<id>[^&#]+)'
     _TESTS = [
         {
             'url': 'http://www.patreon.com/creation?hid=743933',
@@ -65,9 +65,7 @@ class PatreonIE(InfoExtractor):
     '''
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         title = self._og_search_title(webpage).strip()
 
@@ -80,11 +78,10 @@ class PatreonIE(InfoExtractor):
             uploader = self._html_search_regex(
                 r'<strong>(.*?)</strong> is creating', webpage, 'uploader')
         else:
-            playlist_js = self._search_regex(
+            playlist = self._parse_json(self._search_regex(
                 r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])',
-                webpage, 'playlist JSON')
-            playlist_json = js_to_json(playlist_js)
-            playlist = json.loads(playlist_json)
+                webpage, 'playlist JSON'),
+                video_id, transform_source=js_to_json)
             data = playlist[0]
             video_url = self._proto_relative_url(data['mp3'])
             thumbnail = self._proto_relative_url(data.get('cover'))

From 6b961a85fdef8219913b7b97cb33ef8d59d15535 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:04:19 +0100
Subject: [PATCH 945/974] [patreon] Add support for embedlies (fixes #4969)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/embedly.py  | 16 ++++++++++++++++
 youtube_dl/extractor/patreon.py  | 26 +++++++++++++++++++++++---
 3 files changed, 40 insertions(+), 3 deletions(-)
 create mode 100644 youtube_dl/extractor/embedly.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c712b907f..84a7edffc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -121,6 +121,7 @@ from .ellentv import (
     EllenTVClipsIE,
 )
 from .elpais import ElPaisIE
+from .embedly import EmbedlyIE
 from .empflix import EMPFlixIE
 from .engadget import EngadgetIE
 from .eporner import EpornerIE
diff --git a/youtube_dl/extractor/embedly.py b/youtube_dl/extractor/embedly.py
new file mode 100644
index 000000000..1cdb11e34
--- /dev/null
+++ b/youtube_dl/extractor/embedly.py
@@ -0,0 +1,16 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+
+
+class EmbedlyIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www|cdn\.)?embedly\.com/widgets/media\.html\?(?:[^#]*?&)?url=(?P<id>[^#&]+)'
+    _TESTS = [{
+        'url': 'https://cdn.embedly.com/widgets/media.html?src=http%3A%2F%2Fwww.youtube.com%2Fembed%2Fvideoseries%3Flist%3DUUGLim4T2loE5rwCMdpCIPVg&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DSU4fj_aEMVw%26list%3DUUGLim4T2loE5rwCMdpCIPVg&image=http%3A%2F%2Fi.ytimg.com%2Fvi%2FSU4fj_aEMVw%2Fhqdefault.jpg&key=8ee8a2e6a8cc47aab1a5ee67f9a178e0&type=text%2Fhtml&schema=youtube&autoplay=1',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        return self.url_result(compat_urllib_parse_unquote(self._match_id(url)))
diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index e5c9822c5..f179ea200 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -1,9 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     js_to_json,
@@ -35,6 +32,23 @@ class PatreonIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*$',
             },
         },
+        {
+            'url': 'https://www.patreon.com/creation?hid=1682498',
+            'info_dict': {
+                'id': 'SU4fj_aEMVw',
+                'ext': 'mp4',
+                'title': 'I\'m on Patreon!',
+                'uploader': 'TraciJHines',
+                'thumbnail': 're:^https?://.*$',
+                'upload_date': '20150211',
+                'description': 'md5:c5a706b1f687817a3de09db1eb93acd4',
+                'uploader_id': 'TraciJHines',
+            },
+            'params': {
+                'noplaylist': True,
+                'skip_download': True,
+            }
+        }
     ]
 
     # Currently Patreon exposes download URL via hidden CSS, so login is not
@@ -72,11 +86,17 @@ class PatreonIE(InfoExtractor):
         attach_fn = self._html_search_regex(
             r'<div class="attach"><a target="_blank" href="([^"]+)">',
             webpage, 'attachment URL', default=None)
+        embed = self._html_search_regex(
+            r'<div id="watchCreation">\s*<iframe class="embedly-embed" src="([^"]+)"',
+            webpage, 'embedded URL', default=None)
+
         if attach_fn is not None:
             video_url = 'http://www.patreon.com' + attach_fn
             thumbnail = self._og_search_thumbnail(webpage)
             uploader = self._html_search_regex(
                 r'<strong>(.*?)</strong> is creating', webpage, 'uploader')
+        elif embed is not None:
+            return self.url_result(embed)
         else:
             playlist = self._parse_json(self._search_regex(
                 r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])',

From 7bd15b1a0378cff6960cd55e40819ef3967f747c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:04:24 +0100
Subject: [PATCH 946/974] release 2015.02.19.1

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e28d7d47c..1fcec086a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -121,6 +121,7 @@
  - **EllenTV**
  - **EllenTV:clips**
  - **ElPais**: El País
+ - **Embedly**
  - **EMPFlix**
  - **Engadget**
  - **Eporner**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8e6ff49d0..db0ed70da 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19'
+__version__ = '2015.02.19.1'

From 1c2528c8a3c599740558d5d752dfb368c8714a89 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:22:50 +0100
Subject: [PATCH 947/974] [cbs] Modernize

---
 youtube_dl/extractor/cbs.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index e43756ec6..1ceb9d8d9 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -39,8 +37,7 @@ class CBSIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         real_id = self._search_regex(
             r"video\.settings\.pid\s*=\s*'([^']+)';",

From 3bf5705316adb4e486ec76a9308198b499787947 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:43:20 +0100
Subject: [PATCH 948/974] [imgur] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/imgur.py    | 84 ++++++++++++++++++++++++++++++++
 2 files changed, 85 insertions(+)
 create mode 100644 youtube_dl/extractor/imgur.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 84a7edffc..1d1f07ff5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -205,6 +205,7 @@ from .imdb import (
     ImdbIE,
     ImdbListIE
 )
+from .imgur import ImgurIE
 from .ina import InaIE
 from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
new file mode 100644
index 000000000..16488e0c4
--- /dev/null
+++ b/youtube_dl/extractor/imgur.py
@@ -0,0 +1,84 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+)
+
+
+class ImgurIE(InfoExtractor):
+    _VALID_URL = r'https?://i\.imgur\.com/(?P<id>[a-zA-Z0-9]+)\.(?:mp4|gifv)'
+
+    _TESTS = [{
+        'url': 'https://i.imgur.com/A61SaA1.gifv',
+        'info_dict': {
+            'id': 'A61SaA1',
+            'ext': 'mp4',
+            'title': 'MRW gifv is up and running without any bugs',
+            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        width = int_or_none(self._search_regex(
+            r'<param name="width" value="([0-9]+)"',
+            webpage, 'width', fatal=False))
+        height = int_or_none(self._search_regex(
+            r'<param name="height" value="([0-9]+)"',
+            webpage, 'height', fatal=False))
+
+        formats = []
+        video_elements = self._search_regex(
+            r'(?s)<div class="video-elements">(.*?)</div>',
+            webpage, 'video elements')
+        formats = []
+        for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):
+            formats.append({
+                'format_id': m.group('type').partition('/')[2],
+                'url': self._proto_relative_url(m.group('src')),
+                'ext': mimetype2ext(m.group('type')),
+                'acodec': 'none',
+                'width': width,
+                'height': height,
+                'http_headers': {
+                    'User-Agent': 'youtube-dl (like wget)',
+                },
+            })
+
+        gif_json = self._search_regex(
+            r'(?s)var\s+videoItem\s*=\s*(\{.*?\})',
+            webpage, 'GIF code', fatal=False)
+        if gif_json:
+            gifd = self._parse_json(
+                gif_json, video_id, transform_source=js_to_json)
+            formats.append({
+                'format_id': 'gif',
+                'preference': -10,
+                'width': width,
+                'height': height,
+                'ext': 'gif',
+                'acodec': 'none',
+                'vcodec': 'gif',
+                'container': 'gif',
+                'url': self._proto_relative_url(gifd['gifUrl']),
+                'filesize': gifd.get('size'),
+                'http_headers': {
+                    'User-Agent': 'youtube-dl (like wget)',
+                },
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+            'title': self._og_search_title(webpage),
+        }

From 1ac1af9b4702fa0b921ceb45f6e7b13b35e0a2de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:43:28 +0100
Subject: [PATCH 949/974] release 2015.02.19.2

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1fcec086a..607e958f6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -191,6 +191,7 @@
  - **ign.com**
  - **imdb**: Internet Movie Database trailers
  - **imdb:list**: Internet Movie Database lists
+ - **Imgur**
  - **Ina**
  - **InfoQ**
  - **Instagram**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index db0ed70da..131dd571b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19.1'
+__version__ = '2015.02.19.2'

From bd03ffc16e0e09c1ec6fb5c3f2c3083238af5349 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:58:52 +0100
Subject: [PATCH 950/974] [netzkino] Skip download in test case

Works fine from Germany, but fails from everywhere else
---
 youtube_dl/extractor/netzkino.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/netzkino.py b/youtube_dl/extractor/netzkino.py
index 93567d1e3..bc17e20aa 100644
--- a/youtube_dl/extractor/netzkino.py
+++ b/youtube_dl/extractor/netzkino.py
@@ -29,6 +29,9 @@ class NetzkinoIE(InfoExtractor):
             'timestamp': 1344858571,
             'age_limit': 12,
         },
+        'params': {
+            'skip_download': 'Download only works from Germany',
+        }
     }
 
     def _real_extract(self, url):

From f37e3f99f01df51af649a3544570b27facc5d92c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 02:00:50 +0100
Subject: [PATCH 951/974] [generic] Correct test case

Video has been reuploaded / edited
---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 36a1f65bf..8dce96a64 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -532,7 +532,7 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': 'Mrj4DVp2zeA',
                 'ext': 'mp4',
-                'upload_date': '20150204',
+                'upload_date': '20150212',
                 'uploader': 'The National Archives UK',
                 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
                 'uploader_id': 'NationalArchives08',

From 71d53ace2f5c4637f5d3390298f475bbeb2452f3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 02:04:28 +0100
Subject: [PATCH 952/974] [sockshare] Do not require thumbnail anymore

Thumbnail is not present on the website anymore.
---
 youtube_dl/extractor/sockshare.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/sockshare.py b/youtube_dl/extractor/sockshare.py
index 7d3c0e937..b5fa6f1da 100644
--- a/youtube_dl/extractor/sockshare.py
+++ b/youtube_dl/extractor/sockshare.py
@@ -25,7 +25,6 @@ class SockshareIE(InfoExtractor):
             'id': '437BE28B89D799D7',
             'title': 'big_buck_bunny_720p_surround.avi',
             'ext': 'avi',
-            'thumbnail': 're:^http://.*\.jpg$',
         }
     }
 
@@ -45,7 +44,7 @@ class SockshareIE(InfoExtractor):
             ''', webpage, 'hash')
 
         fields = {
-            "hash": confirm_hash,
+            "hash": confirm_hash.encode('utf-8'),
             "confirm": "Continue as Free User"
         }
 
@@ -68,7 +67,7 @@ class SockshareIE(InfoExtractor):
             webpage, 'title', default=None)
         thumbnail = self._html_search_regex(
             r'<img\s+src="([^"]*)".+?name="bg"',
-            webpage, 'thumbnail')
+            webpage, 'thumbnail', default=None)
 
         formats = [{
             'format_id': 'sd',

From 3d54788495aae464a4ce63b5f093d4409f600dd8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 02:12:08 +0100
Subject: [PATCH 953/974] [webofstories] Fix extraction

---
 youtube_dl/extractor/webofstories.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index 396cf4e83..73077a312 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -45,19 +45,17 @@ class WebOfStoriesIE(InfoExtractor):
         description = self._html_search_meta('description', webpage)
         thumbnail = self._og_search_thumbnail(webpage)
 
-        story_filename = self._search_regex(
-            r'\.storyFileName\("([^"]+)"\)', webpage, 'story filename')
-        speaker_id = self._search_regex(
-            r'\.speakerId\("([^"]+)"\)', webpage, 'speaker ID')
-        story_id = self._search_regex(
-            r'\.storyId\((\d+)\)', webpage, 'story ID')
-        speaker_type = self._search_regex(
-            r'\.speakerType\("([^"]+)"\)', webpage, 'speaker type')
-        great_life = self._search_regex(
-            r'isGreatLifeStory\s*=\s*(true|false)', webpage, 'great life story')
+        embed_params = [s.strip(" \r\n\t'") for s in self._search_regex(
+            r'(?s)\$\("#embedCode"\).html\(getEmbedCode\((.*?)\)',
+            webpage, 'embed params').split(',')]
+
+        (
+            _, speaker_id, story_id, story_duration,
+            speaker_type, great_life, _thumbnail, _has_subtitles,
+            story_filename, _story_order) = embed_params
+
         is_great_life_series = great_life == 'true'
-        duration = int_or_none(self._search_regex(
-            r'\.duration\((\d+)\)', webpage, 'duration', fatal=False))
+        duration = int_or_none(story_duration)
 
         # URL building, see: http://www.webofstories.com/scripts/player.js
         ms_prefix = ''

From 1a13940c8dada638f8298b6c1406f38d4a3bf270 Mon Sep 17 00:00:00 2001
From: John Boehr <jbboehr@gmail.com>
Date: Wed, 18 Feb 2015 18:12:48 -0800
Subject: [PATCH 954/974] [imgur] support regular URL

---
 youtube_dl/extractor/imgur.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 16488e0c4..8449c45f4 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -7,11 +7,11 @@ from ..utils import (
     int_or_none,
     js_to_json,
     mimetype2ext,
+    ExtractorError,
 )
 
-
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://i\.imgur\.com/(?P<id>[a-zA-Z0-9]+)\.(?:mp4|gifv)'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.)?(?:mp4|gifv)?'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -21,12 +21,25 @@ class ImgurIE(InfoExtractor):
             'title': 'MRW gifv is up and running without any bugs',
             'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
         },
+    }, {
+        'url': 'https://imgur.com/A61SaA1',
+        'info_dict': {
+            'id': 'A61SaA1',
+            'ext': 'mp4',
+            'title': 'MRW gifv is up and running without any bugs',
+            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
+        if not sources:
+            raise ExtractorError(
+                'No sources found for video %s' % video_id, expected=True)
+
         width = int_or_none(self._search_regex(
             r'<param name="width" value="([0-9]+)"',
             webpage, 'width', fatal=False))

From 383456aa29c9c47066d422d3fdfdce29da2eee07 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 04:09:52 +0100
Subject: [PATCH 955/974] [Makefile] Also delete *.avi files in clean

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 0636fc4cb..573c82685 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin

From e880c66bd800e4769ffdff41fb9dc88cf3206c12 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 04:12:40 +0100
Subject: [PATCH 956/974] [theonion] Modernize

---
 youtube_dl/extractor/theonion.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/theonion.py b/youtube_dl/extractor/theonion.py
index b65d8e03f..10239c906 100644
--- a/youtube_dl/extractor/theonion.py
+++ b/youtube_dl/extractor/theonion.py
@@ -4,11 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
 
 
 class TheOnionIE(InfoExtractor):
-    _VALID_URL = r'(?x)https?://(?:www\.)?theonion\.com/video/[^,]+,(?P<article_id>[0-9]+)/?'
+    _VALID_URL = r'https?://(?:www\.)?theonion\.com/video/[^,]+,(?P<id>[0-9]+)/?'
     _TEST = {
         'url': 'http://www.theonion.com/video/man-wearing-mm-jacket-gods-image,36918/',
         'md5': '19eaa9a39cf9b9804d982e654dc791ee',
@@ -22,10 +21,8 @@ class TheOnionIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        article_id = mobj.group('article_id')
-
-        webpage = self._download_webpage(url, article_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
 
         video_id = self._search_regex(
             r'"videoId":\s(\d+),', webpage, 'video ID')
@@ -34,10 +31,6 @@ class TheOnionIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
 
         sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
-        if not sources:
-            raise ExtractorError(
-                'No sources found for video %s' % video_id, expected=True)
-
         formats = []
         for src, type_ in sources:
             if type_ == 'video/mp4':
@@ -54,15 +47,15 @@ class TheOnionIE(InfoExtractor):
                 })
             elif type_ == 'application/x-mpegURL':
                 formats.extend(
-                    self._extract_m3u8_formats(src, video_id, preference=-1))
+                    self._extract_m3u8_formats(src, display_id, preference=-1))
             else:
                 self.report_warning(
                     'Encountered unexpected format: %s' % type_)
-
         self._sort_formats(formats)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,

From d236b37ac94cd36657c881e18b8d9187483afa80 Mon Sep 17 00:00:00 2001
From: John Boehr <jbboehr@gmail.com>
Date: Wed, 18 Feb 2015 19:28:19 -0800
Subject: [PATCH 957/974] [imgur] improve regex #4998

---
 youtube_dl/extractor/imgur.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 8449c45f4..38c961773 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -11,7 +11,7 @@ from ..utils import (
 )
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.)?(?:mp4|gifv)?'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.mp4|\.gifv)?'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',

From 9e2d7dca87a15cf455fa6c4843a0241ba0b7ad77 Mon Sep 17 00:00:00 2001
From: John Boehr <jbboehr@gmail.com>
Date: Wed, 18 Feb 2015 19:47:54 -0800
Subject: [PATCH 958/974] [imgur] improve error check for non-video URLs

---
 youtube_dl/extractor/imgur.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 38c961773..7937a5c81 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    str_or_none,
     js_to_json,
     mimetype2ext,
     ExtractorError,
@@ -35,11 +36,6 @@ class ImgurIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
-        if not sources:
-            raise ExtractorError(
-                'No sources found for video %s' % video_id, expected=True)
-
         width = int_or_none(self._search_regex(
             r'<param name="width" value="([0-9]+)"',
             webpage, 'width', fatal=False))
@@ -47,10 +43,13 @@ class ImgurIE(InfoExtractor):
             r'<param name="height" value="([0-9]+)"',
             webpage, 'height', fatal=False))
 
-        formats = []
-        video_elements = self._search_regex(
+        video_elements = str_or_none(self._search_regex(
             r'(?s)<div class="video-elements">(.*?)</div>',
-            webpage, 'video elements')
+            webpage, 'video elements', fatal=False))
+        if not video_elements:
+            raise ExtractorError(
+                'No sources found for video %s' % video_id, expected=True)
+
         formats = []
         for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):
             formats.append({

From b88ba05356bdae07245dd5240b36415002e25d0b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 05:53:09 +0100
Subject: [PATCH 959/974] [imgur] Simplify

---
 youtube_dl/extractor/imgur.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 7937a5c81..b16c7aed0 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -5,12 +5,12 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    str_or_none,
     js_to_json,
     mimetype2ext,
     ExtractorError,
 )
 
+
 class ImgurIE(InfoExtractor):
     _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.mp4|\.gifv)?'
 
@@ -43,12 +43,13 @@ class ImgurIE(InfoExtractor):
             r'<param name="height" value="([0-9]+)"',
             webpage, 'height', fatal=False))
 
-        video_elements = str_or_none(self._search_regex(
+        video_elements = self._search_regex(
             r'(?s)<div class="video-elements">(.*?)</div>',
-            webpage, 'video elements', fatal=False))
+            webpage, 'video elements', default=None)
         if not video_elements:
             raise ExtractorError(
-                'No sources found for video %s' % video_id, expected=True)
+                'No sources found for video %s. Maybe an image?' % video_id,
+                expected=True)
 
         formats = []
         for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):

From a3b9157f499e0ba0ef9cfd44b70b0d43795f60f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 13:06:53 +0100
Subject: [PATCH 960/974] [cbssports] Add extractor (closes #4996)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/cbssports.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 youtube_dl/extractor/cbssports.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1d1f07ff5..c2424ed48 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -58,6 +58,7 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
+from .cbssports import CBSSportsIE
 from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
diff --git a/youtube_dl/extractor/cbssports.py b/youtube_dl/extractor/cbssports.py
new file mode 100644
index 000000000..ae47e74cc
--- /dev/null
+++ b/youtube_dl/extractor/cbssports.py
@@ -0,0 +1,30 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class CBSSportsIE(InfoExtractor):
+    _VALID_URL = r'http://www\.cbssports\.com/video/player/(?P<section>[^/]+)/(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://www.cbssports.com/video/player/tennis/318462531970/0/us-open-flashbacks-1990s',
+        'info_dict': {
+            'id': '_d5_GbO8p1sT',
+            'ext': 'flv',
+            'title': 'US Open flashbacks: 1990s',
+            'description': 'Bill Macatee relives the best moments in US Open history from the 1990s.',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        section = mobj.group('section')
+        video_id = mobj.group('id')
+        all_videos = self._download_json(
+            'http://www.cbssports.com/data/video/player/getVideos/%s?as=json' % section,
+            video_id)
+        # The json file contains the info of all the videos in the section
+        video_info = next(v for v in all_videos if v['pcid'] == video_id)
+        return self.url_result('theplatform:%s' % video_info['pid'], 'ThePlatform')

From 5c8a3f862a325987a9ddbdc7f08ea62aad80278e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 15:00:39 +0100
Subject: [PATCH 961/974] [nbc] Use a test video that works outside the US

---
 youtube_dl/extractor/nbc.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 89a2845fe..3645d3033 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -18,13 +18,13 @@ class NBCIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.nbc.com/chicago-fire/video/i-am-a-firefighter/2734188',
+            'url': 'http://www.nbc.com/the-tonight-show/segments/112966',
             # md5 checksum is not stable
             'info_dict': {
-                'id': 'bTmnLCvIbaaH',
+                'id': 'c9xnCo0YPOPH',
                 'ext': 'flv',
-                'title': 'I Am a Firefighter',
-                'description': 'An emergency puts Dawson\'sf irefighter skills to the ultimate test in this four-part digital series.',
+                'title': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
+                'description': 'Jimmy gives out free scoops of his new "Tonight Dough" ice cream flavor by surprising customers at the Ben & Jerry\'s scoop shop.',
             },
         },
         {

From d5c69f1da41f3e370c500482303196ae845472ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Feb 2015 21:47:11 +0600
Subject: [PATCH 962/974] [5min] Cover joystiq.com URLs (Closes #4962)

---
 youtube_dl/extractor/fivemin.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 5b24b921c..157094e8c 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -14,6 +14,7 @@ class FiveMinIE(InfoExtractor):
     IE_NAME = '5min'
     _VALID_URL = r'''(?x)
         (?:https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(?:.*?&)?playList=|
+            https?://(?:(?:massively|www)\.)?joystiq\.com/video/|
             5min:)
         (?P<id>\d+)
         '''

From e66e1a0046ca804461c4c9c3e890165ec9d865e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Feb 2015 22:15:19 +0600
Subject: [PATCH 963/974] [pornhub] Add support for playlists (Closes #4995)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/pornhub.py  | 30 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c2424ed48..9b5855768 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -353,7 +353,10 @@ from .playfm import PlayFMIE
 from .playvid import PlayvidIE
 from .podomatic import PodomaticIE
 from .pornhd import PornHdIE
-from .pornhub import PornHubIE
+from .pornhub import (
+    PornHubIE,
+    PornHubPlaylistIE,
+)
 from .pornotube import PornotubeIE
 from .pornoxo import PornoXOIE
 from .promptfile import PromptFileIE
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index fb2032832..913995044 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -110,3 +110,33 @@ class PornHubIE(InfoExtractor):
             'formats': formats,
             'age_limit': 18,
         }
+
+
+class PornHubPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/playlist/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.pornhub.com/playlist/6201671',
+        'info_dict': {
+            'id': '6201671',
+            'title': 'P0p4',
+        },
+        'playlist_mincount': 35,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
+            for video_url in set(re.findall('href="/?(view_video\.php\?viewkey=\d+[^"]*)"', webpage))
+        ]
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'playlistObject\s*=\s*({.+?});', webpage, 'playlist'),
+            playlist_id)
+
+        return self.playlist_result(
+            entries, playlist_id, playlist.get('title'), playlist.get('description'))

From 8fc642eb5b69f25d0488509c4c4f9d8bfe2b7225 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Feb 2015 22:15:49 +0600
Subject: [PATCH 964/974] [pornhub] Fix uploader regex

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 913995044..3a27e3789 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -56,7 +56,7 @@ class PornHubIE(InfoExtractor):
 
         video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
         video_uploader = self._html_search_regex(
-            r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|<span class="username)[^>]+>(.+?)<',
+            r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|span class="username)[^>]+>(.+?)<',
             webpage, 'uploader', fatal=False)
         thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
         if thumbnail:

From 6140baf4e1aa7015ac6b43e93236ac6a5813569e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 18:17:31 +0100
Subject: [PATCH 965/974] [nationalgeographic] Add extractor (closes #4960)

---
 youtube_dl/extractor/__init__.py           |  1 +
 youtube_dl/extractor/nationalgeographic.py | 38 ++++++++++++++++++++++
 youtube_dl/extractor/theplatform.py        |  4 ++-
 3 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/nationalgeographic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9b5855768..f225ac654 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -285,6 +285,7 @@ from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
+from .nationalgeographic import NationalGeographicIE
 from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
new file mode 100644
index 000000000..c18640c5a
--- /dev/null
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    smuggle_url,
+    url_basename,
+)
+
+
+class NationalGeographicIE(InfoExtractor):
+    _VALID_URL = r'http://video\.nationalgeographic\.com/video/.*?'
+
+    _TEST = {
+        'url': 'http://video.nationalgeographic.com/video/news/150210-news-crab-mating-vin?source=featuredvideo',
+        'info_dict': {
+            'id': '4DmDACA6Qtk_',
+            'ext': 'flv',
+            'title': 'Mating Crabs Busted by Sharks',
+            'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
+        },
+        'add_ie': ['ThePlatform'],
+    }
+
+    def _real_extract(self, url):
+        name = url_basename(url)
+
+        webpage = self._download_webpage(url, name)
+        feed_url = self._search_regex(r'data-feed-url="([^"]+)"', webpage, 'feed url')
+        guid = self._search_regex(r'data-video-guid="([^"]+)"', webpage, 'guid')
+
+        feed = self._download_xml('%s?byGuid=%s' % (feed_url, guid), name)
+        content = feed.find('.//{http://search.yahoo.com/mrss/}content')
+        theplatform_id = url_basename(content.attrib.get('url'))
+
+        return self.url_result(smuggle_url(
+            'http://link.theplatform.com/s/ngs/%s?format=SMIL&formats=MPEG4&manifest=f4m' % theplatform_id,
+            # For some reason, the normal links don't work and we must force the use of f4m
+            {'force_smil_url': True}))
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 1579822f2..f7b34bd26 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -71,7 +71,9 @@ class ThePlatformIE(SubtitlesInfoExtractor):
         if not provider_id:
             provider_id = 'dJ5BDC'
 
-        if mobj.group('config'):
+        if smuggled_data.get('force_smil_url', False):
+            smil_url = url
+        elif mobj.group('config'):
             config_url = url + '&form=json'
             config_url = config_url.replace('swf/', 'config/')
             config_url = config_url.replace('onsite/', 'onsite/config/')

From a21420389edf665bfeb9610888d2d39dae374945 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 19:28:17 +0100
Subject: [PATCH 966/974] release 2015.02.19.3

---
 docs/supportedsites.md | 3 +++
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 607e958f6..f6ba28e7a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -68,6 +68,7 @@
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
  - **CBS**
  - **CBSNews**: CBS News
+ - **CBSSports**
  - **CeskaTelevize**
  - **channel9**: Channel 9
  - **Chilloutzone**
@@ -264,6 +265,7 @@
  - **myvideo**
  - **MyVidster**
  - **n-tv.de**
+ - **NationalGeographic**
  - **Naver**
  - **NBA**
  - **NBC**
@@ -321,6 +323,7 @@
  - **podomatic**
  - **PornHd**
  - **PornHub**
+ - **PornHubPlaylist**
  - **Pornotube**
  - **PornoXO**
  - **PromptFile**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 131dd571b..537e8cf60 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19.2'
+__version__ = '2015.02.19.3'

From dd0a58f5f028980ba76450998b218ea7a4920420 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Feb 2015 01:19:38 +0600
Subject: [PATCH 967/974] [blinkx] Fix extraction

---
 youtube_dl/extractor/blinkx.py | 49 ++++++++++++++++------------------
 1 file changed, 23 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/blinkx.py b/youtube_dl/extractor/blinkx.py
index 3e461e715..3b8eabe8f 100644
--- a/youtube_dl/extractor/blinkx.py
+++ b/youtube_dl/extractor/blinkx.py
@@ -1,40 +1,35 @@
 from __future__ import unicode_literals
 
 import json
-import re
 
 from .common import InfoExtractor
-from ..utils import remove_start
+from ..utils import (
+    remove_start,
+    int_or_none,
+)
 
 
 class BlinkxIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
+    _VALID_URL = r'(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
     IE_NAME = 'blinkx'
 
     _TEST = {
-        'url': 'http://www.blinkx.com/ce/8aQUy7GVFYgFzpKhT0oqsilwOGFRVXk3R1ZGWWdGenBLaFQwb3FzaWx3OGFRVXk3R1ZGWWdGenB',
-        'md5': '2e9a07364af40163a908edbf10bb2492',
+        'url': 'http://www.blinkx.com/ce/Da0Gw3xc5ucpNduzLuDDlv4WC9PuI4fDi1-t6Y3LyfdY2SZS5Urbvn-UPJvrvbo8LTKTc67Wu2rPKSQDJyZeeORCR8bYkhs8lI7eqddznH2ofh5WEEdjYXnoRtj7ByQwt7atMErmXIeYKPsSDuMAAqJDlQZ-3Ff4HJVeH_s3Gh8oQ',
+        'md5': '337cf7a344663ec79bf93a526a2e06c7',
         'info_dict': {
-            'id': '8aQUy7GV',
+            'id': 'Da0Gw3xc',
             'ext': 'mp4',
-            'title': 'Police Car Rolls Away',
-            'uploader': 'stupidvideos.com',
-            'upload_date': '20131215',
-            'timestamp': 1387068000,
-            'description': 'A police car gently rolls away from a fight. Maybe it felt weird being around a confrontation and just had to get out of there!',
-            'duration': 14.886,
-            'thumbnails': [{
-                'width': 100,
-                'height': 76,
-                'resolution': '100x76',
-                'url': 'http://cdn.blinkx.com/stream/b/41/StupidVideos/20131215/1873969261/1873969261_tn_0.jpg',
-            }],
+            'title': 'No Daily Show for John Oliver; HBO Show Renewed - IGN News',
+            'uploader': 'IGN News',
+            'upload_date': '20150217',
+            'timestamp': 1424215740,
+            'description': 'HBO has renewed Last Week Tonight With John Oliver for two more seasons.',
+            'duration': 47.743333,
         },
     }
 
-    def _real_extract(self, rl):
-        m = re.match(self._VALID_URL, rl)
-        video_id = m.group('id')
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
         display_id = video_id[:8]
 
         api_url = ('https://apib4.blinkx.com/api.php?action=play_video&' +
@@ -60,18 +55,20 @@ class BlinkxIE(InfoExtractor):
             elif m['type'] in ('flv', 'mp4'):
                 vcodec = remove_start(m['vcodec'], 'ff')
                 acodec = remove_start(m['acodec'], 'ff')
-                tbr = (int(m['vbr']) + int(m['abr'])) // 1000
+                vbr = int_or_none(m.get('vbr') or m.get('vbitrate'), 1000)
+                abr = int_or_none(m.get('abr') or m.get('abitrate'), 1000)
+                tbr = vbr + abr if vbr and abr else None
                 format_id = '%s-%sk-%s' % (vcodec, tbr, m['w'])
                 formats.append({
                     'format_id': format_id,
                     'url': m['link'],
                     'vcodec': vcodec,
                     'acodec': acodec,
-                    'abr': int(m['abr']) // 1000,
-                    'vbr': int(m['vbr']) // 1000,
+                    'abr': abr,
+                    'vbr': vbr,
                     'tbr': tbr,
-                    'width': int(m['w']),
-                    'height': int(m['h']),
+                    'width': int_or_none(m.get('w')),
+                    'height': int_or_none(m.get('h')),
                 })
 
         self._sort_formats(formats)

From 5e9a033e6e8054605bb87c8448f95a5bb86a71bb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 20 Feb 2015 02:52:03 +0100
Subject: [PATCH 968/974] [imgur] Allow alternative values

Every now and then, imgur.com goes crazy and gives us a generic title and description (otherwise it looks all fine though).
Simply update the test case to allow for that craziness.
---
 youtube_dl/extractor/imgur.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index b16c7aed0..fe5d95e2c 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -19,16 +19,16 @@ class ImgurIE(InfoExtractor):
         'info_dict': {
             'id': 'A61SaA1',
             'ext': 'mp4',
-            'title': 'MRW gifv is up and running without any bugs',
-            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+            'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
+            'description': 're:The origin of the Internet\'s most viral images$|The Internet\'s visual storytelling community\. Explore, share, and discuss the best visual stories the Internet has to offer\.$',
         },
     }, {
         'url': 'https://imgur.com/A61SaA1',
         'info_dict': {
             'id': 'A61SaA1',
             'ext': 'mp4',
-            'title': 'MRW gifv is up and running without any bugs',
-            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+            'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
+            'description': 're:The origin of the Internet\'s most viral images$|The Internet\'s visual storytelling community\. Explore, share, and discuss the best visual stories the Internet has to offer\.$',
         },
     }]
 

From 50efb383f01071fe5ae6ec6da2557031da54abde Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 20 Feb 2015 14:13:41 +0200
Subject: [PATCH 969/974] [tv4] Add new extractor (Closes #4839) (Closes #2103)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/tv4.py      | 100 +++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100644 youtube_dl/extractor/tv4.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f225ac654..7d0c6b5ac 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -490,6 +490,7 @@ from .tumblr import TumblrIE
 from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
+from .tv4 import TV4IE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
new file mode 100644
index 000000000..1c4b6d635
--- /dev/null
+++ b/youtube_dl/extractor/tv4.py
@@ -0,0 +1,100 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+)
+
+
+class TV4IE(InfoExtractor):
+    IE_DESC = 'tv4.se and tv4play.se'
+    _VALID_URL = r'''(?x)https?://(?:www\.)?
+        (?:
+            tv4\.se/(?:[^/]+)/klipp/(?:.*)-|
+            tv4play\.se/
+            (?:
+                (?:program|barn)/(?:[^\?]+)\?video_id=|
+                iframe/video/|
+                film/|
+                sport/|
+            )
+        )(?P<id>[0-9]+)'''
+    _TESTS = [
+        {
+            'url': 'http://www.tv4.se/kalla-fakta/klipp/kalla-fakta-5-english-subtitles-2491650',
+            'md5': '909d6454b87b10a25aa04c4bdd416a9b',
+            'info_dict': {
+                'id': '2491650',
+                'ext': 'mp4',
+                'title': 'Kalla Fakta 5 (english subtitles)',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'timestamp': int,
+                'upload_date': '20131125',
+            },
+        },
+        {
+            'url': 'http://www.tv4play.se/iframe/video/3054113',
+            'md5': '77f851c55139ffe0ebd41b6a5552489b',
+            'info_dict': {
+                'id': '3054113',
+                'ext': 'mp4',
+                'title': 'Så här jobbar ficktjuvarna - se avslöjande bilder',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'Unika bilder avslöjar hur turisternas fickor vittjas mitt på Stockholms central. Två experter på ficktjuvarna avslöjar knepen du ska se upp för.',
+                'timestamp': int,
+                'upload_date': '20150130',
+            },
+        },
+        {
+            'url': 'http://www.tv4play.se/sport/3060959',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.tv4play.se/film/2378136',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.tv4play.se/barn/looney-tunes?video_id=3062412',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        info = self._download_json(
+            'http://www.tv4play.se/player/assets/%s.json' % video_id, video_id, 'Downloading video info JSON')
+
+        # If is_geo_restricted is true, it doesn't neceserally mean we can't download it
+        if info['is_geo_restricted']:
+            self.report_warning('This content might not be available in your country due to licensing restrictions.')
+        if info['requires_subscription']:
+            raise ExtractorError('This content requires subscription.', expected=True)
+
+        sources_data = self._download_json(
+            'https://prima.tv4play.se/api/web/asset/%s/play.json?protocol=http&videoFormat=MP4' % video_id, video_id, 'Downloading sources JSON')
+        sources = sources_data['playback']
+
+        formats = []
+        for item in sources.get('items', {}).get('item', []):
+            ext, bitrate = item['mediaFormat'], item['bitrate']
+            formats.append({
+                'format_id': '%s_%s' % (ext, bitrate),
+                'tbr': bitrate,
+                'ext': ext,
+                'url': item['url'],
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['title'],
+            'formats': formats,
+            'description': info.get('description'),
+            'timestamp': parse_iso8601(info.get('broadcast_date_time')),
+            'duration': info.get('duration'),
+            'thumbnail': info.get('image'),
+            'is_live': sources.get('live'),
+        }

From 1bd838608fda2da789c8991ec7fc194ed5ec0a1f Mon Sep 17 00:00:00 2001
From: minusf <minusf@gmail.com>
Date: Fri, 20 Feb 2015 18:24:20 +0100
Subject: [PATCH 970/974] prefer 'code' to 'uri' if present

---
 youtube_dl/extractor/ted.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 10b3b706a..5e5322974 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -134,9 +134,13 @@ class TEDIE(SubtitlesInfoExtractor):
 
         if talk_info.get('external') is not None:
             self.to_screen('Found video from %s' % talk_info['external']['service'])
+            if 'code' in talk_info['external']:
+                ext_url = talk_info['external']['code']
+            else:
+                ext_url = talk_info['external']['uri']
             return {
                 '_type': 'url',
-                'url': talk_info['external']['uri'],
+                'url': ext_url,
             }
 
         formats = [{

From a461a11989e147c30e3611005861cc63dcd16af3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Feb 2015 00:14:38 +0600
Subject: [PATCH 971/974] [ted] Improve external video handling and add test

---
 youtube_dl/extractor/ted.py | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 5e5322974..59678399d 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -83,6 +83,22 @@ class TEDIE(SubtitlesInfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # YouTube video
+        'url': 'http://www.ted.com/talks/jeffrey_kluger_the_sibling_bond',
+        'add_ie': ['Youtube'],
+        'info_dict': {
+            'id': 'aFBIPO-P7LM',
+            'ext': 'mp4',
+            'title': 'The hidden power of siblings: Jeff Kluger at TEDxAsheville',
+            'description': 'md5:3d7a4f50d95ca5dd67104e2a20f43fe1',
+            'uploader': 'TEDx Talks',
+            'uploader_id': 'TEDxTalks',
+            'upload_date': '20111216',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     _NATIVE_FORMATS = {
@@ -132,15 +148,16 @@ class TEDIE(SubtitlesInfoExtractor):
 
         talk_info = self._extract_info(webpage)['talks'][0]
 
-        if talk_info.get('external') is not None:
-            self.to_screen('Found video from %s' % talk_info['external']['service'])
-            if 'code' in talk_info['external']:
-                ext_url = talk_info['external']['code']
-            else:
-                ext_url = talk_info['external']['uri']
+        external = talk_info.get('external')
+        if external:
+            service = external['service']
+            self.to_screen('Found video from %s' % service)
+            ext_url = None
+            if service.lower() == 'youtube':
+                ext_url = external.get('code')
             return {
                 '_type': 'url',
-                'url': ext_url,
+                'url': ext_url or external['uri'],
             }
 
         formats = [{

From 3da0db62e6c4122b151349e7a05f59803da4fbbc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 20 Feb 2015 23:22:47 +0100
Subject: [PATCH 972/974] [escapist] Fix extraction (fixes #5017)

---
 youtube_dl/extractor/escapist.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 4303feccd..6b693b3b6 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -22,6 +22,7 @@ class EscapistIE(InfoExtractor):
             'uploader_id': 'the-escapist-presents',
             'uploader': 'The Escapist Presents',
             'title': "Breaking Down Baldur's Gate",
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
@@ -40,9 +41,8 @@ class EscapistIE(InfoExtractor):
         raw_title = self._html_search_meta('title', webpage, fatal=True)
         title = raw_title.partition(' : ')[2]
 
-        player_url = self._og_search_video_url(webpage, name='player URL')
-        config_url = compat_urllib_parse.unquote(self._search_regex(
-            r'config=(.*)$', player_url, 'config URL'))
+        config_url = compat_urllib_parse.unquote(self._html_search_regex(
+            r'<param name="flashvars" value="config=([^"&]+)', webpage, 'config URL'))
 
         formats = []
 
@@ -81,5 +81,4 @@ class EscapistIE(InfoExtractor):
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': description,
-            'player_url': player_url,
         }

From 1e9a9e167d5e518f44402394bb4cc23712ff32fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 20 Feb 2015 23:23:12 +0100
Subject: [PATCH 973/974] release 2015.02.20

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f6ba28e7a..dbff5e270 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -451,6 +451,7 @@
  - **Turbo**
  - **Tutv**
  - **tv.dfb.de**
+ - **TV4**: tv4.se and tv4play.se
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
  - **tvp.pl:Series**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 537e8cf60..9fd0ee963 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19.3'
+__version__ = '2015.02.20'

From a00a8bcc8a131e0bea605ab3cfa0f09bedde9e0c Mon Sep 17 00:00:00 2001
From: CyberJacob <cyberjacob@gmail.com>
Date: Fri, 20 Feb 2015 22:43:51 +0000
Subject: [PATCH 974/974] Change example URLs in readme (fixes #5018y)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a2c148311..8ea31d605 100644
--- a/README.md
+++ b/README.md
@@ -571,7 +571,7 @@ Support requests for services that **do** purchase the rights to distribute thei
 
 ### How can I detect whether a given URL is supported by youtube-dl?
 
-For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
 It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.